1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BootTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_CompareAndSwapB:
1512 case Op_CompareAndSwapS:
1513 case Op_GetAndSetI:
1514 case Op_GetAndSetL:
1515 case Op_GetAndSetP:
1516 case Op_GetAndSetN:
1517 case Op_GetAndAddI:
1518 case Op_GetAndAddL:
1519 return true;
1520 case Op_CompareAndExchangeI:
1521 case Op_CompareAndExchangeN:
1522 case Op_CompareAndExchangeB:
1523 case Op_CompareAndExchangeS:
1524 case Op_CompareAndExchangeL:
1525 case Op_CompareAndExchangeP:
1526 case Op_WeakCompareAndSwapB:
1527 case Op_WeakCompareAndSwapS:
1528 case Op_WeakCompareAndSwapI:
1529 case Op_WeakCompareAndSwapL:
1530 case Op_WeakCompareAndSwapP:
1531 case Op_WeakCompareAndSwapN:
1532 return maybe_volatile;
1533 default:
1534 return false;
1535 }
1536 }
1537
1538 // helper to determine the maximum number of Phi nodes we may need to
1539 // traverse when searching from a card mark membar for the merge mem
1540 // feeding a trailing membar or vice versa
1541
1542 // predicates controlling emit of ldr<x>/ldar<x>
1543
1544 bool unnecessary_acquire(const Node *barrier)
1545 {
1546 assert(barrier->is_MemBar(), "expecting a membar");
1547
1548 MemBarNode* mb = barrier->as_MemBar();
1549
1550 if (mb->trailing_load()) {
1551 return true;
1552 }
1553
1554 if (mb->trailing_load_store()) {
1555 Node* load_store = mb->in(MemBarNode::Precedent);
1556 assert(load_store->is_LoadStore(), "unexpected graph shape");
1557 return is_CAS(load_store->Opcode(), true);
1558 }
1559
1560 return false;
1561 }
1562
1563 bool needs_acquiring_load(const Node *n)
1564 {
1565 assert(n->is_Load(), "expecting a load");
1566 LoadNode *ld = n->as_Load();
1567 return ld->is_acquire();
1568 }
1569
1570 bool unnecessary_release(const Node *n)
1571 {
1572 assert((n->is_MemBar() &&
1573 n->Opcode() == Op_MemBarRelease),
1574 "expecting a release membar");
1575
1576 MemBarNode *barrier = n->as_MemBar();
1577 if (!barrier->leading()) {
1578 return false;
1579 } else {
1580 Node* trailing = barrier->trailing_membar();
1581 MemBarNode* trailing_mb = trailing->as_MemBar();
1582 assert(trailing_mb->trailing(), "Not a trailing membar?");
1583 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1584
1585 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1586 if (mem->is_Store()) {
1587 assert(mem->as_Store()->is_release(), "");
1588 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1589 return true;
1590 } else {
1591 assert(mem->is_LoadStore(), "");
1592 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1593 return is_CAS(mem->Opcode(), true);
1594 }
1595 }
1596 return false;
1597 }
1598
1599 bool unnecessary_volatile(const Node *n)
1600 {
1601 // assert n->is_MemBar();
1602 MemBarNode *mbvol = n->as_MemBar();
1603
1604 bool release = mbvol->trailing_store();
1605 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1606 #ifdef ASSERT
1607 if (release) {
1608 Node* leading = mbvol->leading_membar();
1609 assert(leading->Opcode() == Op_MemBarRelease, "");
1610 assert(leading->as_MemBar()->leading_store(), "");
1611 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1612 }
1613 #endif
1614
1615 return release;
1616 }
1617
1618 // predicates controlling emit of str<x>/stlr<x>
1619
1620 bool needs_releasing_store(const Node *n)
1621 {
1622 // assert n->is_Store();
1623 StoreNode *st = n->as_Store();
1624 return st->trailing_membar() != nullptr;
1625 }
1626
1627 // predicate controlling translation of CAS
1628 //
1629 // returns true if CAS needs to use an acquiring load otherwise false
1630
1631 bool needs_acquiring_load_exclusive(const Node *n)
1632 {
1633 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1634 LoadStoreNode* ldst = n->as_LoadStore();
1635 if (is_CAS(n->Opcode(), false)) {
1636 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1637 } else {
1638 return ldst->trailing_membar() != nullptr;
1639 }
1640
1641 // so we can just return true here
1642 return true;
1643 }
1644
1645 #define __ masm->
1646
1647 // advance declarations for helper functions to convert register
1648 // indices to register objects
1649
1650 // the ad file has to provide implementations of certain methods
1651 // expected by the generic code
1652 //
1653 // REQUIRED FUNCTIONALITY
1654
1655 //=============================================================================
1656
1657 // !!!!! Special hack to get all types of calls to specify the byte offset
1658 // from the start of the call to the point where the return address
1659 // will point.
1660
1661 int MachCallStaticJavaNode::ret_addr_offset()
1662 {
1663 // call should be a simple bl
1664 int off = 4;
1665 return off;
1666 }
1667
1668 int MachCallDynamicJavaNode::ret_addr_offset()
1669 {
1670 return 16; // movz, movk, movk, bl
1671 }
1672
1673 int MachCallRuntimeNode::ret_addr_offset() {
1674 // for generated stubs the call will be
1675 // bl(addr)
1676 // or with far branches
1677 // bl(trampoline_stub)
1678 // for real runtime callouts it will be six instructions
1679 // see aarch64_enc_java_to_runtime
1680 // adr(rscratch2, retaddr)
1681 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1682 // lea(rscratch1, RuntimeAddress(addr)
1683 // blr(rscratch1)
1684 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1685 if (cb) {
1686 return 1 * NativeInstruction::instruction_size;
1687 } else {
1688 return 6 * NativeInstruction::instruction_size;
1689 }
1690 }
1691
1692 //=============================================================================
1693
1694 #ifndef PRODUCT
1695 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1696 st->print("BREAKPOINT");
1697 }
1698 #endif
1699
1700 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1701 __ brk(0);
1702 }
1703
1704 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1705 return MachNode::size(ra_);
1706 }
1707
1708 //=============================================================================
1709
1710 #ifndef PRODUCT
1711 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1712 st->print("nop \t# %d bytes pad for loops and calls", _count);
1713 }
1714 #endif
1715
1716 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1717 for (int i = 0; i < _count; i++) {
1718 __ nop();
1719 }
1720 }
1721
1722 uint MachNopNode::size(PhaseRegAlloc*) const {
1723 return _count * NativeInstruction::instruction_size;
1724 }
1725
1726 //=============================================================================
1727 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1728
1729 int ConstantTable::calculate_table_base_offset() const {
1730 return 0; // absolute addressing, no offset
1731 }
1732
1733 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1734 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1735 ShouldNotReachHere();
1736 }
1737
1738 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1739 // Empty encoding
1740 }
1741
1742 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1743 return 0;
1744 }
1745
1746 #ifndef PRODUCT
1747 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1748 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1749 }
1750 #endif
1751
1752 #ifndef PRODUCT
1753 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1754 Compile* C = ra_->C;
1755
1756 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1757
1758 if (C->output()->need_stack_bang(framesize))
1759 st->print("# stack bang size=%d\n\t", framesize);
1760
1761 if (VM_Version::use_rop_protection()) {
1762 st->print("ldr zr, [lr]\n\t");
1763 st->print("paciaz\n\t");
1764 }
1765 if (framesize < ((1 << 9) + 2 * wordSize)) {
1766 st->print("sub sp, sp, #%d\n\t", framesize);
1767 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1768 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1769 } else {
1770 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1771 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1772 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1773 st->print("sub sp, sp, rscratch1");
1774 }
1775 if (C->stub_function() == nullptr) {
1776 st->print("\n\t");
1777 st->print("ldr rscratch1, [guard]\n\t");
1778 st->print("dmb ishld\n\t");
1779 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1780 st->print("cmp rscratch1, rscratch2\n\t");
1781 st->print("b.eq skip");
1782 st->print("\n\t");
1783 st->print("blr #nmethod_entry_barrier_stub\n\t");
1784 st->print("b skip\n\t");
1785 st->print("guard: int\n\t");
1786 st->print("\n\t");
1787 st->print("skip:\n\t");
1788 }
1789 }
1790 #endif
1791
1792 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1793 Compile* C = ra_->C;
1794
1795 // n.b. frame size includes space for return pc and rfp
1796 const int framesize = C->output()->frame_size_in_bytes();
1797
1798 if (C->clinit_barrier_on_entry()) {
1799 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1800
1801 Label L_skip_barrier;
1802
1803 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1804 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1805 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1806 __ bind(L_skip_barrier);
1807 }
1808
1809 if (C->max_vector_size() > 0) {
1810 __ reinitialize_ptrue();
1811 }
1812
1813 int bangsize = C->output()->bang_size_in_bytes();
1814 if (C->output()->need_stack_bang(bangsize))
1815 __ generate_stack_overflow_check(bangsize);
1816
1817 __ build_frame(framesize);
1818
1819 if (C->stub_function() == nullptr) {
1820 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1821 // Dummy labels for just measuring the code size
1822 Label dummy_slow_path;
1823 Label dummy_continuation;
1824 Label dummy_guard;
1825 Label* slow_path = &dummy_slow_path;
1826 Label* continuation = &dummy_continuation;
1827 Label* guard = &dummy_guard;
1828 if (!Compile::current()->output()->in_scratch_emit_size()) {
1829 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1830 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1831 Compile::current()->output()->add_stub(stub);
1832 slow_path = &stub->entry();
1833 continuation = &stub->continuation();
1834 guard = &stub->guard();
1835 }
1836 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1837 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1838 }
1839
1840 if (VerifyStackAtCalls) {
1841 Unimplemented();
1842 }
1843
1844 C->output()->set_frame_complete(__ offset());
1845
1846 if (C->has_mach_constant_base_node()) {
1847 // NOTE: We set the table base offset here because users might be
1848 // emitted before MachConstantBaseNode.
1849 ConstantTable& constant_table = C->output()->constant_table();
1850 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1851 }
1852 }
1853
1854 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1855 {
1856 return MachNode::size(ra_); // too many variables; just compute it
1857 // the hard way
1858 }
1859
1860 int MachPrologNode::reloc() const
1861 {
1862 return 0;
1863 }
1864
1865 //=============================================================================
1866
1867 #ifndef PRODUCT
1868 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1869 Compile* C = ra_->C;
1870 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1871
1872 st->print("# pop frame %d\n\t",framesize);
1873
1874 if (framesize == 0) {
1875 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1876 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1877 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1878 st->print("add sp, sp, #%d\n\t", framesize);
1879 } else {
1880 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1881 st->print("add sp, sp, rscratch1\n\t");
1882 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1883 }
1884 if (VM_Version::use_rop_protection()) {
1885 st->print("autiaz\n\t");
1886 st->print("ldr zr, [lr]\n\t");
1887 }
1888
1889 if (do_polling() && C->is_method_compilation()) {
1890 st->print("# test polling word\n\t");
1891 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1892 st->print("cmp sp, rscratch1\n\t");
1893 st->print("bhi #slow_path");
1894 }
1895 }
1896 #endif
1897
1898 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1899 Compile* C = ra_->C;
1900 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1901
1902 __ remove_frame(framesize);
1903
1904 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1905 __ reserved_stack_check();
1906 }
1907
1908 if (do_polling() && C->is_method_compilation()) {
1909 Label dummy_label;
1910 Label* code_stub = &dummy_label;
1911 if (!C->output()->in_scratch_emit_size()) {
1912 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1913 C->output()->add_stub(stub);
1914 code_stub = &stub->entry();
1915 }
1916 __ relocate(relocInfo::poll_return_type);
1917 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1918 }
1919 }
1920
1921 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1922 // Variable size. Determine dynamically.
1923 return MachNode::size(ra_);
1924 }
1925
1926 int MachEpilogNode::reloc() const {
1927 // Return number of relocatable values contained in this instruction.
1928 return 1; // 1 for polling page.
1929 }
1930
1931 const Pipeline * MachEpilogNode::pipeline() const {
1932 return MachNode::pipeline_class();
1933 }
1934
1935 //=============================================================================
1936
1937 static enum RC rc_class(OptoReg::Name reg) {
1938
1939 if (reg == OptoReg::Bad) {
1940 return rc_bad;
1941 }
1942
1943 // we have 32 int registers * 2 halves
1944 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1945
1946 if (reg < slots_of_int_registers) {
1947 return rc_int;
1948 }
1949
1950 // we have 32 float register * 8 halves
1951 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1952 if (reg < slots_of_int_registers + slots_of_float_registers) {
1953 return rc_float;
1954 }
1955
1956 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1957 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1958 return rc_predicate;
1959 }
1960
1961 // Between predicate regs & stack is the flags.
1962 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1963
1964 return rc_stack;
1965 }
1966
1967 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1968 Compile* C = ra_->C;
1969
1970 // Get registers to move.
1971 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1972 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1973 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1974 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1975
1976 enum RC src_hi_rc = rc_class(src_hi);
1977 enum RC src_lo_rc = rc_class(src_lo);
1978 enum RC dst_hi_rc = rc_class(dst_hi);
1979 enum RC dst_lo_rc = rc_class(dst_lo);
1980
1981 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1982
1983 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1984 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1985 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1986 "expected aligned-adjacent pairs");
1987 }
1988
1989 if (src_lo == dst_lo && src_hi == dst_hi) {
1990 return 0; // Self copy, no move.
1991 }
1992
1993 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1994 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1995 int src_offset = ra_->reg2offset(src_lo);
1996 int dst_offset = ra_->reg2offset(dst_lo);
1997
1998 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1999 uint ireg = ideal_reg();
2000 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2001 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2002 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2003 if (ireg == Op_VecA && masm) {
2004 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2005 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2006 // stack->stack
2007 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2008 sve_vector_reg_size_in_bytes);
2009 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2010 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2011 sve_vector_reg_size_in_bytes);
2012 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2013 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2016 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]),
2018 as_FloatRegister(Matcher::_regEncode[src_lo]));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 } else if (masm) {
2023 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2024 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2025 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2026 // stack->stack
2027 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2028 if (ireg == Op_VecD) {
2029 __ unspill(rscratch1, true, src_offset);
2030 __ spill(rscratch1, true, dst_offset);
2031 } else {
2032 __ spill_copy128(src_offset, dst_offset);
2033 }
2034 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2035 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2036 ireg == Op_VecD ? __ T8B : __ T16B,
2037 as_FloatRegister(Matcher::_regEncode[src_lo]));
2038 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2039 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2040 ireg == Op_VecD ? __ D : __ Q,
2041 ra_->reg2offset(dst_lo));
2042 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2043 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2044 ireg == Op_VecD ? __ D : __ Q,
2045 ra_->reg2offset(src_lo));
2046 } else {
2047 ShouldNotReachHere();
2048 }
2049 }
2050 } else if (masm) {
2051 switch (src_lo_rc) {
2052 case rc_int:
2053 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2054 if (is64) {
2055 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2056 as_Register(Matcher::_regEncode[src_lo]));
2057 } else {
2058 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2059 as_Register(Matcher::_regEncode[src_lo]));
2060 }
2061 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2062 if (is64) {
2063 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2064 as_Register(Matcher::_regEncode[src_lo]));
2065 } else {
2066 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2067 as_Register(Matcher::_regEncode[src_lo]));
2068 }
2069 } else { // gpr --> stack spill
2070 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2071 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2072 }
2073 break;
2074 case rc_float:
2075 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2076 if (is64) {
2077 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2078 as_FloatRegister(Matcher::_regEncode[src_lo]));
2079 } else {
2080 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2081 as_FloatRegister(Matcher::_regEncode[src_lo]));
2082 }
2083 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2084 if (is64) {
2085 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2086 as_FloatRegister(Matcher::_regEncode[src_lo]));
2087 } else {
2088 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2089 as_FloatRegister(Matcher::_regEncode[src_lo]));
2090 }
2091 } else { // fpr --> stack spill
2092 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2093 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2094 is64 ? __ D : __ S, dst_offset);
2095 }
2096 break;
2097 case rc_stack:
2098 if (dst_lo_rc == rc_int) { // stack --> gpr load
2099 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2100 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2101 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2102 is64 ? __ D : __ S, src_offset);
2103 } else if (dst_lo_rc == rc_predicate) {
2104 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2105 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2106 } else { // stack --> stack copy
2107 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2108 if (ideal_reg() == Op_RegVectMask) {
2109 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2110 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2111 } else {
2112 __ unspill(rscratch1, is64, src_offset);
2113 __ spill(rscratch1, is64, dst_offset);
2114 }
2115 }
2116 break;
2117 case rc_predicate:
2118 if (dst_lo_rc == rc_predicate) {
2119 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2120 } else if (dst_lo_rc == rc_stack) {
2121 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2122 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2123 } else {
2124 assert(false, "bad src and dst rc_class combination.");
2125 ShouldNotReachHere();
2126 }
2127 break;
2128 default:
2129 assert(false, "bad rc_class for spill");
2130 ShouldNotReachHere();
2131 }
2132 }
2133
2134 if (st) {
2135 st->print("spill ");
2136 if (src_lo_rc == rc_stack) {
2137 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2138 } else {
2139 st->print("%s -> ", Matcher::regName[src_lo]);
2140 }
2141 if (dst_lo_rc == rc_stack) {
2142 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2143 } else {
2144 st->print("%s", Matcher::regName[dst_lo]);
2145 }
2146 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2147 int vsize = 0;
2148 switch (ideal_reg()) {
2149 case Op_VecD:
2150 vsize = 64;
2151 break;
2152 case Op_VecX:
2153 vsize = 128;
2154 break;
2155 case Op_VecA:
2156 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2157 break;
2158 default:
2159 assert(false, "bad register type for spill");
2160 ShouldNotReachHere();
2161 }
2162 st->print("\t# vector spill size = %d", vsize);
2163 } else if (ideal_reg() == Op_RegVectMask) {
2164 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2165 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2166 st->print("\t# predicate spill size = %d", vsize);
2167 } else {
2168 st->print("\t# spill size = %d", is64 ? 64 : 32);
2169 }
2170 }
2171
2172 return 0;
2173
2174 }
2175
2176 #ifndef PRODUCT
2177 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2178 if (!ra_)
2179 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2180 else
2181 implementation(nullptr, ra_, false, st);
2182 }
2183 #endif
2184
2185 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2186 implementation(masm, ra_, false, nullptr);
2187 }
2188
2189 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2190 return MachNode::size(ra_);
2191 }
2192
2193 //=============================================================================
2194
2195 #ifndef PRODUCT
2196 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2197 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2198 int reg = ra_->get_reg_first(this);
2199 st->print("add %s, rsp, #%d]\t# box lock",
2200 Matcher::regName[reg], offset);
2201 }
2202 #endif
2203
2204 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2205 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2206 int reg = ra_->get_encode(this);
2207
2208 // This add will handle any 24-bit signed offset. 24 bits allows an
2209 // 8 megabyte stack frame.
2210 __ add(as_Register(reg), sp, offset);
2211 }
2212
2213 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2214 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2215 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2216
2217 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2218 return NativeInstruction::instruction_size;
2219 } else {
2220 return 2 * NativeInstruction::instruction_size;
2221 }
2222 }
2223
2224 //=============================================================================
2225
2226 #ifndef PRODUCT
2227 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2228 {
2229 st->print_cr("# MachUEPNode");
2230 if (UseCompressedClassPointers) {
2231 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2232 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2233 st->print_cr("\tcmpw rscratch1, r10");
2234 } else {
2235 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2236 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2237 st->print_cr("\tcmp rscratch1, r10");
2238 }
2239 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2240 }
2241 #endif
2242
2243 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2244 {
2245 __ ic_check(InteriorEntryAlignment);
2246 }
2247
2248 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2249 {
2250 return MachNode::size(ra_);
2251 }
2252
2253 // REQUIRED EMIT CODE
2254
2255 //=============================================================================
2256
2257 // Emit deopt handler code.
2258 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2259 {
2260 // Note that the code buffer's insts_mark is always relative to insts.
2261 // That's why we must use the macroassembler to generate a handler.
2262 address base = __ start_a_stub(size_deopt_handler());
2263 if (base == nullptr) {
2264 ciEnv::current()->record_failure("CodeCache is full");
2265 return 0; // CodeBuffer::expand failed
2266 }
2267
2268 int offset = __ offset();
2269 Label start;
2270 __ bind(start);
2271 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2272
2273 int entry_offset = __ offset();
2274 __ b(start);
2275
2276 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2277 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2278 "out of bounds read in post-call NOP check");
2279 __ end_a_stub();
2280 return entry_offset;
2281 }
2282
2283 // REQUIRED MATCHER CODE
2284
2285 //=============================================================================
2286
2287 bool Matcher::match_rule_supported(int opcode) {
2288 if (!has_match_rule(opcode))
2289 return false;
2290
2291 switch (opcode) {
2292 case Op_OnSpinWait:
2293 return VM_Version::supports_on_spin_wait();
2294 case Op_CacheWB:
2295 case Op_CacheWBPreSync:
2296 case Op_CacheWBPostSync:
2297 if (!VM_Version::supports_data_cache_line_flush()) {
2298 return false;
2299 }
2300 break;
2301 case Op_ExpandBits:
2302 case Op_CompressBits:
2303 if (!VM_Version::supports_svebitperm()) {
2304 return false;
2305 }
2306 break;
2307 case Op_FmaF:
2308 case Op_FmaD:
2309 case Op_FmaVF:
2310 case Op_FmaVD:
2311 if (!UseFMA) {
2312 return false;
2313 }
2314 break;
2315 case Op_FmaHF:
2316 // UseFMA flag also needs to be checked along with FEAT_FP16
2317 if (!UseFMA || !is_feat_fp16_supported()) {
2318 return false;
2319 }
2320 break;
2321 case Op_AddHF:
2322 case Op_SubHF:
2323 case Op_MulHF:
2324 case Op_DivHF:
2325 case Op_MinHF:
2326 case Op_MaxHF:
2327 case Op_SqrtHF:
2328 // Half-precision floating point scalar operations require FEAT_FP16
2329 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2330 // features are supported.
2331 if (!is_feat_fp16_supported()) {
2332 return false;
2333 }
2334 break;
2335 }
2336
2337 return true; // Per default match rules are supported.
2338 }
2339
2340 const RegMask* Matcher::predicate_reg_mask(void) {
2341 return &_PR_REG_mask;
2342 }
2343
2344 bool Matcher::supports_vector_calling_convention(void) {
2345 return EnableVectorSupport;
2346 }
2347
2348 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2349 assert(EnableVectorSupport, "sanity");
2350 int lo = V0_num;
2351 int hi = V0_H_num;
2352 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2353 hi = V0_K_num;
2354 }
2355 return OptoRegPair(hi, lo);
2356 }
2357
2358 // Is this branch offset short enough that a short branch can be used?
2359 //
2360 // NOTE: If the platform does not provide any short branch variants, then
2361 // this method should return false for offset 0.
2362 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2363 // The passed offset is relative to address of the branch.
2364
2365 return (-32768 <= offset && offset < 32768);
2366 }
2367
2368 // Vector width in bytes.
2369 int Matcher::vector_width_in_bytes(BasicType bt) {
2370 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2371 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2372 // Minimum 2 values in vector
2373 if (size < 2*type2aelembytes(bt)) size = 0;
2374 // But never < 4
2375 if (size < 4) size = 0;
2376 return size;
2377 }
2378
2379 // Limits on vector size (number of elements) loaded into vector.
2380 int Matcher::max_vector_size(const BasicType bt) {
2381 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2382 }
2383
2384 int Matcher::min_vector_size(const BasicType bt) {
2385 // Usually, the shortest vector length supported by AArch64 ISA and
2386 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2387 // vectors in a few special cases.
2388 int size;
2389 switch(bt) {
2390 case T_BOOLEAN:
2391 // Load/store a vector mask with only 2 elements for vector types
2392 // such as "2I/2F/2L/2D".
2393 size = 2;
2394 break;
2395 case T_BYTE:
2396 // Generate a "4B" vector, to support vector cast between "8B/16B"
2397 // and "4S/4I/4L/4F/4D".
2398 size = 4;
2399 break;
2400 case T_SHORT:
2401 // Generate a "2S" vector, to support vector cast between "4S/8S"
2402 // and "2I/2L/2F/2D".
2403 size = 2;
2404 break;
2405 default:
2406 // Limit the min vector length to 64-bit.
2407 size = 8 / type2aelembytes(bt);
2408 // The number of elements in a vector should be at least 2.
2409 size = MAX2(size, 2);
2410 }
2411
2412 int max_size = max_vector_size(bt);
2413 return MIN2(size, max_size);
2414 }
2415
2416 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2417 return Matcher::max_vector_size(bt);
2418 }
2419
2420 // Actual max scalable vector register length.
2421 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2422 return Matcher::max_vector_size(bt);
2423 }
2424
2425 // Vector ideal reg.
2426 uint Matcher::vector_ideal_reg(int len) {
2427 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2428 return Op_VecA;
2429 }
2430 switch(len) {
2431 // For 16-bit/32-bit mask vector, reuse VecD.
2432 case 2:
2433 case 4:
2434 case 8: return Op_VecD;
2435 case 16: return Op_VecX;
2436 }
2437 ShouldNotReachHere();
2438 return 0;
2439 }
2440
2441 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2442 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2443 switch (ideal_reg) {
2444 case Op_VecA: return new vecAOper();
2445 case Op_VecD: return new vecDOper();
2446 case Op_VecX: return new vecXOper();
2447 }
2448 ShouldNotReachHere();
2449 return nullptr;
2450 }
2451
2452 bool Matcher::is_reg2reg_move(MachNode* m) {
2453 return false;
2454 }
2455
2456 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2457 return false;
2458 }
2459
2460 bool Matcher::is_generic_vector(MachOper* opnd) {
2461 return opnd->opcode() == VREG;
2462 }
2463
2464 // Return whether or not this register is ever used as an argument.
2465 // This function is used on startup to build the trampoline stubs in
2466 // generateOptoStub. Registers not mentioned will be killed by the VM
2467 // call in the trampoline, and arguments in those registers not be
2468 // available to the callee.
2469 bool Matcher::can_be_java_arg(int reg)
2470 {
2471 return
2472 reg == R0_num || reg == R0_H_num ||
2473 reg == R1_num || reg == R1_H_num ||
2474 reg == R2_num || reg == R2_H_num ||
2475 reg == R3_num || reg == R3_H_num ||
2476 reg == R4_num || reg == R4_H_num ||
2477 reg == R5_num || reg == R5_H_num ||
2478 reg == R6_num || reg == R6_H_num ||
2479 reg == R7_num || reg == R7_H_num ||
2480 reg == V0_num || reg == V0_H_num ||
2481 reg == V1_num || reg == V1_H_num ||
2482 reg == V2_num || reg == V2_H_num ||
2483 reg == V3_num || reg == V3_H_num ||
2484 reg == V4_num || reg == V4_H_num ||
2485 reg == V5_num || reg == V5_H_num ||
2486 reg == V6_num || reg == V6_H_num ||
2487 reg == V7_num || reg == V7_H_num;
2488 }
2489
2490 bool Matcher::is_spillable_arg(int reg)
2491 {
2492 return can_be_java_arg(reg);
2493 }
2494
2495 uint Matcher::int_pressure_limit()
2496 {
2497 // JDK-8183543: When taking the number of available registers as int
2498 // register pressure threshold, the jtreg test:
2499 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2500 // failed due to C2 compilation failure with
2501 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2502 //
2503 // A derived pointer is live at CallNode and then is flagged by RA
2504 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2505 // derived pointers and lastly fail to spill after reaching maximum
2506 // number of iterations. Lowering the default pressure threshold to
2507 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2508 // a high register pressure area of the code so that split_DEF can
2509 // generate DefinitionSpillCopy for the derived pointer.
2510 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2511 if (!PreserveFramePointer) {
2512 // When PreserveFramePointer is off, frame pointer is allocatable,
2513 // but different from other SOC registers, it is excluded from
2514 // fatproj's mask because its save type is No-Save. Decrease 1 to
2515 // ensure high pressure at fatproj when PreserveFramePointer is off.
2516 // See check_pressure_at_fatproj().
2517 default_int_pressure_threshold--;
2518 }
2519 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2520 }
2521
2522 uint Matcher::float_pressure_limit()
2523 {
2524 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2525 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2526 }
2527
2528 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2529 return false;
2530 }
2531
2532 const RegMask& Matcher::divI_proj_mask() {
2533 ShouldNotReachHere();
2534 return RegMask::EMPTY;
2535 }
2536
2537 // Register for MODI projection of divmodI.
2538 const RegMask& Matcher::modI_proj_mask() {
2539 ShouldNotReachHere();
2540 return RegMask::EMPTY;
2541 }
2542
2543 // Register for DIVL projection of divmodL.
2544 const RegMask& Matcher::divL_proj_mask() {
2545 ShouldNotReachHere();
2546 return RegMask::EMPTY;
2547 }
2548
2549 // Register for MODL projection of divmodL.
2550 const RegMask& Matcher::modL_proj_mask() {
2551 ShouldNotReachHere();
2552 return RegMask::EMPTY;
2553 }
2554
2555 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2556 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2557 Node* u = addp->fast_out(i);
2558 if (u->is_LoadStore()) {
2559 // On AArch64, LoadStoreNodes (i.e. compare and swap
2560 // instructions) only take register indirect as an operand, so
2561 // any attempt to use an AddPNode as an input to a LoadStoreNode
2562 // must fail.
2563 return false;
2564 }
2565 if (u->is_Mem()) {
2566 int opsize = u->as_Mem()->memory_size();
2567 assert(opsize > 0, "unexpected memory operand size");
2568 if (u->as_Mem()->memory_size() != (1<<shift)) {
2569 return false;
2570 }
2571 }
2572 }
2573 return true;
2574 }
2575
2576 // Convert BootTest condition to Assembler condition.
2577 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2578 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2579 Assembler::Condition result;
2580 switch(cond) {
2581 case BoolTest::eq:
2582 result = Assembler::EQ; break;
2583 case BoolTest::ne:
2584 result = Assembler::NE; break;
2585 case BoolTest::le:
2586 result = Assembler::LE; break;
2587 case BoolTest::ge:
2588 result = Assembler::GE; break;
2589 case BoolTest::lt:
2590 result = Assembler::LT; break;
2591 case BoolTest::gt:
2592 result = Assembler::GT; break;
2593 case BoolTest::ule:
2594 result = Assembler::LS; break;
2595 case BoolTest::uge:
2596 result = Assembler::HS; break;
2597 case BoolTest::ult:
2598 result = Assembler::LO; break;
2599 case BoolTest::ugt:
2600 result = Assembler::HI; break;
2601 case BoolTest::overflow:
2602 result = Assembler::VS; break;
2603 case BoolTest::no_overflow:
2604 result = Assembler::VC; break;
2605 default:
2606 ShouldNotReachHere();
2607 return Assembler::Condition(-1);
2608 }
2609
2610 // Check conversion
2611 if (cond & BoolTest::unsigned_compare) {
2612 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2613 } else {
2614 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2615 }
2616
2617 return result;
2618 }
2619
2620 // Binary src (Replicate con)
2621 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2622 if (n == nullptr || m == nullptr) {
2623 return false;
2624 }
2625
2626 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2627 return false;
2628 }
2629
2630 Node* imm_node = m->in(1);
2631 if (!imm_node->is_Con()) {
2632 return false;
2633 }
2634
2635 const Type* t = imm_node->bottom_type();
2636 if (!(t->isa_int() || t->isa_long())) {
2637 return false;
2638 }
2639
2640 switch (n->Opcode()) {
2641 case Op_AndV:
2642 case Op_OrV:
2643 case Op_XorV: {
2644 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2645 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2646 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2647 }
2648 case Op_AddVB:
2649 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2650 case Op_AddVS:
2651 case Op_AddVI:
2652 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2653 case Op_AddVL:
2654 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2655 default:
2656 return false;
2657 }
2658 }
2659
2660 // (XorV src (Replicate m1))
2661 // (XorVMask src (MaskAll m1))
2662 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2663 if (n != nullptr && m != nullptr) {
2664 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2665 VectorNode::is_all_ones_vector(m);
2666 }
2667 return false;
2668 }
2669
2670 // Should the matcher clone input 'm' of node 'n'?
2671 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2672 if (is_vshift_con_pattern(n, m) ||
2673 is_vector_bitwise_not_pattern(n, m) ||
2674 is_valid_sve_arith_imm_pattern(n, m) ||
2675 is_encode_and_store_pattern(n, m)) {
2676 mstack.push(m, Visit);
2677 return true;
2678 }
2679 return false;
2680 }
2681
2682 // Should the Matcher clone shifts on addressing modes, expecting them
2683 // to be subsumed into complex addressing expressions or compute them
2684 // into registers?
2685 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2686
2687 // Loads and stores with indirect memory input (e.g., volatile loads and
2688 // stores) do not subsume the input into complex addressing expressions. If
2689 // the addressing expression is input to at least one such load or store, do
2690 // not clone the addressing expression. Query needs_acquiring_load and
2691 // needs_releasing_store as a proxy for indirect memory input, as it is not
2692 // possible to directly query for indirect memory input at this stage.
2693 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2694 Node* n = m->fast_out(i);
2695 if (n->is_Load() && needs_acquiring_load(n)) {
2696 return false;
2697 }
2698 if (n->is_Store() && needs_releasing_store(n)) {
2699 return false;
2700 }
2701 }
2702
2703 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2704 return true;
2705 }
2706
2707 Node *off = m->in(AddPNode::Offset);
2708 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2709 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2710 // Are there other uses besides address expressions?
2711 !is_visited(off)) {
2712 address_visited.set(off->_idx); // Flag as address_visited
2713 mstack.push(off->in(2), Visit);
2714 Node *conv = off->in(1);
2715 if (conv->Opcode() == Op_ConvI2L &&
2716 // Are there other uses besides address expressions?
2717 !is_visited(conv)) {
2718 address_visited.set(conv->_idx); // Flag as address_visited
2719 mstack.push(conv->in(1), Pre_Visit);
2720 } else {
2721 mstack.push(conv, Pre_Visit);
2722 }
2723 address_visited.test_set(m->_idx); // Flag as address_visited
2724 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2725 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2726 return true;
2727 } else if (off->Opcode() == Op_ConvI2L &&
2728 // Are there other uses besides address expressions?
2729 !is_visited(off)) {
2730 address_visited.test_set(m->_idx); // Flag as address_visited
2731 address_visited.set(off->_idx); // Flag as address_visited
2732 mstack.push(off->in(1), Pre_Visit);
2733 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2734 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2735 return true;
2736 }
2737 return false;
2738 }
2739
2740 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2741 { \
2742 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2743 guarantee(DISP == 0, "mode not permitted for volatile"); \
2744 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2745 __ INSN(REG, as_Register(BASE)); \
2746 }
2747
2748
2749 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2750 {
2751 Address::extend scale;
2752
2753 // Hooboy, this is fugly. We need a way to communicate to the
2754 // encoder that the index needs to be sign extended, so we have to
2755 // enumerate all the cases.
2756 switch (opcode) {
2757 case INDINDEXSCALEDI2L:
2758 case INDINDEXSCALEDI2LN:
2759 case INDINDEXI2L:
2760 case INDINDEXI2LN:
2761 scale = Address::sxtw(size);
2762 break;
2763 default:
2764 scale = Address::lsl(size);
2765 }
2766
2767 if (index == -1) {
2768 return Address(base, disp);
2769 } else {
2770 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2771 return Address(base, as_Register(index), scale);
2772 }
2773 }
2774
2775
2776 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2777 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2778 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2779 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2780 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2781
2782 // Used for all non-volatile memory accesses. The use of
2783 // $mem->opcode() to discover whether this pattern uses sign-extended
2784 // offsets is something of a kludge.
2785 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2786 Register reg, int opcode,
2787 Register base, int index, int scale, int disp,
2788 int size_in_memory)
2789 {
2790 Address addr = mem2address(opcode, base, index, scale, disp);
2791 if (addr.getMode() == Address::base_plus_offset) {
2792 /* Fix up any out-of-range offsets. */
2793 assert_different_registers(rscratch1, base);
2794 assert_different_registers(rscratch1, reg);
2795 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2796 }
2797 (masm->*insn)(reg, addr);
2798 }
2799
2800 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2801 FloatRegister reg, int opcode,
2802 Register base, int index, int size, int disp,
2803 int size_in_memory)
2804 {
2805 Address::extend scale;
2806
2807 switch (opcode) {
2808 case INDINDEXSCALEDI2L:
2809 case INDINDEXSCALEDI2LN:
2810 scale = Address::sxtw(size);
2811 break;
2812 default:
2813 scale = Address::lsl(size);
2814 }
2815
2816 if (index == -1) {
2817 // Fix up any out-of-range offsets.
2818 assert_different_registers(rscratch1, base);
2819 Address addr = Address(base, disp);
2820 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2821 (masm->*insn)(reg, addr);
2822 } else {
2823 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2824 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2825 }
2826 }
2827
2828 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2829 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2830 int opcode, Register base, int index, int size, int disp)
2831 {
2832 if (index == -1) {
2833 (masm->*insn)(reg, T, Address(base, disp));
2834 } else {
2835 assert(disp == 0, "unsupported address mode");
2836 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2837 }
2838 }
2839
2840 %}
2841
2842
2843
2844 //----------ENCODING BLOCK-----------------------------------------------------
2845 // This block specifies the encoding classes used by the compiler to
2846 // output byte streams. Encoding classes are parameterized macros
2847 // used by Machine Instruction Nodes in order to generate the bit
2848 // encoding of the instruction. Operands specify their base encoding
2849 // interface with the interface keyword. There are currently
2850 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2851 // COND_INTER. REG_INTER causes an operand to generate a function
2852 // which returns its register number when queried. CONST_INTER causes
2853 // an operand to generate a function which returns the value of the
2854 // constant when queried. MEMORY_INTER causes an operand to generate
2855 // four functions which return the Base Register, the Index Register,
2856 // the Scale Value, and the Offset Value of the operand when queried.
2857 // COND_INTER causes an operand to generate six functions which return
2858 // the encoding code (ie - encoding bits for the instruction)
2859 // associated with each basic boolean condition for a conditional
2860 // instruction.
2861 //
2862 // Instructions specify two basic values for encoding. Again, a
2863 // function is available to check if the constant displacement is an
2864 // oop. They use the ins_encode keyword to specify their encoding
2865 // classes (which must be a sequence of enc_class names, and their
2866 // parameters, specified in the encoding block), and they use the
2867 // opcode keyword to specify, in order, their primary, secondary, and
2868 // tertiary opcode. Only the opcode sections which a particular
2869 // instruction needs for encoding need to be specified.
2870 encode %{
2871 // Build emit functions for each basic byte or larger field in the
2872 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2873 // from C++ code in the enc_class source block. Emit functions will
2874 // live in the main source block for now. In future, we can
2875 // generalize this by adding a syntax that specifies the sizes of
2876 // fields in an order, so that the adlc can build the emit functions
2877 // automagically
2878
2879 // catch all for unimplemented encodings
2880 enc_class enc_unimplemented %{
2881 __ unimplemented("C2 catch all");
2882 %}
2883
2884 // BEGIN Non-volatile memory access
2885
2886 // This encoding class is generated automatically from ad_encode.m4.
2887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2888 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2889 Register dst_reg = as_Register($dst$$reg);
2890 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2891 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2892 %}
2893
2894 // This encoding class is generated automatically from ad_encode.m4.
2895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2896 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2897 Register dst_reg = as_Register($dst$$reg);
2898 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2899 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2900 %}
2901
2902 // This encoding class is generated automatically from ad_encode.m4.
2903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2904 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2905 Register dst_reg = as_Register($dst$$reg);
2906 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2907 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2908 %}
2909
2910 // This encoding class is generated automatically from ad_encode.m4.
2911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2912 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2913 Register dst_reg = as_Register($dst$$reg);
2914 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2915 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2916 %}
2917
2918 // This encoding class is generated automatically from ad_encode.m4.
2919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2920 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2921 Register dst_reg = as_Register($dst$$reg);
2922 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2923 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2924 %}
2925
2926 // This encoding class is generated automatically from ad_encode.m4.
2927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2928 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2929 Register dst_reg = as_Register($dst$$reg);
2930 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2931 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2932 %}
2933
2934 // This encoding class is generated automatically from ad_encode.m4.
2935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2936 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2937 Register dst_reg = as_Register($dst$$reg);
2938 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2939 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2940 %}
2941
2942 // This encoding class is generated automatically from ad_encode.m4.
2943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2944 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2945 Register dst_reg = as_Register($dst$$reg);
2946 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2947 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2948 %}
2949
2950 // This encoding class is generated automatically from ad_encode.m4.
2951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2952 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2953 Register dst_reg = as_Register($dst$$reg);
2954 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2955 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2956 %}
2957
2958 // This encoding class is generated automatically from ad_encode.m4.
2959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2960 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2961 Register dst_reg = as_Register($dst$$reg);
2962 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2963 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2964 %}
2965
2966 // This encoding class is generated automatically from ad_encode.m4.
2967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2968 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2969 Register dst_reg = as_Register($dst$$reg);
2970 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2971 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2972 %}
2973
2974 // This encoding class is generated automatically from ad_encode.m4.
2975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2976 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2977 Register dst_reg = as_Register($dst$$reg);
2978 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2979 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2980 %}
2981
2982 // This encoding class is generated automatically from ad_encode.m4.
2983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2984 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2985 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2986 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2987 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2988 %}
2989
2990 // This encoding class is generated automatically from ad_encode.m4.
2991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2992 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2993 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2994 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2995 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2996 %}
2997
2998 // This encoding class is generated automatically from ad_encode.m4.
2999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3000 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3001 Register src_reg = as_Register($src$$reg);
3002 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3003 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3004 %}
3005
3006 // This encoding class is generated automatically from ad_encode.m4.
3007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3008 enc_class aarch64_enc_strb0(memory1 mem) %{
3009 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3010 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3011 %}
3012
3013 // This encoding class is generated automatically from ad_encode.m4.
3014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3015 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3016 Register src_reg = as_Register($src$$reg);
3017 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3018 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3019 %}
3020
3021 // This encoding class is generated automatically from ad_encode.m4.
3022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3023 enc_class aarch64_enc_strh0(memory2 mem) %{
3024 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3025 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3026 %}
3027
3028 // This encoding class is generated automatically from ad_encode.m4.
3029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3030 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3031 Register src_reg = as_Register($src$$reg);
3032 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3033 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3034 %}
3035
3036 // This encoding class is generated automatically from ad_encode.m4.
3037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3038 enc_class aarch64_enc_strw0(memory4 mem) %{
3039 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3040 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3041 %}
3042
3043 // This encoding class is generated automatically from ad_encode.m4.
3044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3045 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3046 Register src_reg = as_Register($src$$reg);
3047 // we sometimes get asked to store the stack pointer into the
3048 // current thread -- we cannot do that directly on AArch64
3049 if (src_reg == r31_sp) {
3050 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3051 __ mov(rscratch2, sp);
3052 src_reg = rscratch2;
3053 }
3054 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3055 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3056 %}
3057
3058 // This encoding class is generated automatically from ad_encode.m4.
3059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3060 enc_class aarch64_enc_str0(memory8 mem) %{
3061 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3062 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3063 %}
3064
3065 // This encoding class is generated automatically from ad_encode.m4.
3066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3067 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3068 FloatRegister src_reg = as_FloatRegister($src$$reg);
3069 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3070 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3071 %}
3072
3073 // This encoding class is generated automatically from ad_encode.m4.
3074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3075 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3076 FloatRegister src_reg = as_FloatRegister($src$$reg);
3077 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3078 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3079 %}
3080
3081 // This encoding class is generated automatically from ad_encode.m4.
3082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3083 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3084 __ membar(Assembler::StoreStore);
3085 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3086 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3087 %}
3088
3089 // END Non-volatile memory access
3090
3091 // Vector loads and stores
3092 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3093 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3094 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3095 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3096 %}
3097
3098 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3099 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3100 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3101 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3102 %}
3103
3104 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3105 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3106 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3107 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3108 %}
3109
3110 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3111 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3112 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3113 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3114 %}
3115
3116 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3117 FloatRegister src_reg = as_FloatRegister($src$$reg);
3118 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3119 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3120 %}
3121
3122 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3123 FloatRegister src_reg = as_FloatRegister($src$$reg);
3124 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3125 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3126 %}
3127
3128 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3129 FloatRegister src_reg = as_FloatRegister($src$$reg);
3130 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3131 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3132 %}
3133
3134 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3135 FloatRegister src_reg = as_FloatRegister($src$$reg);
3136 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3137 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3138 %}
3139
3140 // volatile loads and stores
3141
3142 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3143 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3144 rscratch1, stlrb);
3145 %}
3146
3147 enc_class aarch64_enc_stlrb0(memory mem) %{
3148 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, stlrb);
3150 %}
3151
3152 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3153 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, stlrh);
3155 %}
3156
3157 enc_class aarch64_enc_stlrh0(memory mem) %{
3158 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3159 rscratch1, stlrh);
3160 %}
3161
3162 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3163 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, stlrw);
3165 %}
3166
3167 enc_class aarch64_enc_stlrw0(memory mem) %{
3168 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, stlrw);
3170 %}
3171
3172 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3173 Register dst_reg = as_Register($dst$$reg);
3174 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3175 rscratch1, ldarb);
3176 __ sxtbw(dst_reg, dst_reg);
3177 %}
3178
3179 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3180 Register dst_reg = as_Register($dst$$reg);
3181 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3182 rscratch1, ldarb);
3183 __ sxtb(dst_reg, dst_reg);
3184 %}
3185
3186 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3187 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3188 rscratch1, ldarb);
3189 %}
3190
3191 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3192 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3193 rscratch1, ldarb);
3194 %}
3195
3196 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3197 Register dst_reg = as_Register($dst$$reg);
3198 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3199 rscratch1, ldarh);
3200 __ sxthw(dst_reg, dst_reg);
3201 %}
3202
3203 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3204 Register dst_reg = as_Register($dst$$reg);
3205 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3206 rscratch1, ldarh);
3207 __ sxth(dst_reg, dst_reg);
3208 %}
3209
3210 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3211 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, ldarh);
3213 %}
3214
3215 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3216 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3217 rscratch1, ldarh);
3218 %}
3219
3220 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3221 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3222 rscratch1, ldarw);
3223 %}
3224
3225 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3226 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3227 rscratch1, ldarw);
3228 %}
3229
3230 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3231 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, ldar);
3233 %}
3234
3235 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3236 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3237 rscratch1, ldarw);
3238 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3239 %}
3240
3241 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3242 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3243 rscratch1, ldar);
3244 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3245 %}
3246
3247 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3248 Register src_reg = as_Register($src$$reg);
3249 // we sometimes get asked to store the stack pointer into the
3250 // current thread -- we cannot do that directly on AArch64
3251 if (src_reg == r31_sp) {
3252 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3253 __ mov(rscratch2, sp);
3254 src_reg = rscratch2;
3255 }
3256 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3257 rscratch1, stlr);
3258 %}
3259
3260 enc_class aarch64_enc_stlr0(memory mem) %{
3261 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3262 rscratch1, stlr);
3263 %}
3264
3265 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3266 {
3267 FloatRegister src_reg = as_FloatRegister($src$$reg);
3268 __ fmovs(rscratch2, src_reg);
3269 }
3270 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3271 rscratch1, stlrw);
3272 %}
3273
3274 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3275 {
3276 FloatRegister src_reg = as_FloatRegister($src$$reg);
3277 __ fmovd(rscratch2, src_reg);
3278 }
3279 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3280 rscratch1, stlr);
3281 %}
3282
3283 // synchronized read/update encodings
3284
3285 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3286 Register dst_reg = as_Register($dst$$reg);
3287 Register base = as_Register($mem$$base);
3288 int index = $mem$$index;
3289 int scale = $mem$$scale;
3290 int disp = $mem$$disp;
3291 if (index == -1) {
3292 if (disp != 0) {
3293 __ lea(rscratch1, Address(base, disp));
3294 __ ldaxr(dst_reg, rscratch1);
3295 } else {
3296 // TODO
3297 // should we ever get anything other than this case?
3298 __ ldaxr(dst_reg, base);
3299 }
3300 } else {
3301 Register index_reg = as_Register(index);
3302 if (disp == 0) {
3303 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3304 __ ldaxr(dst_reg, rscratch1);
3305 } else {
3306 __ lea(rscratch1, Address(base, disp));
3307 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3308 __ ldaxr(dst_reg, rscratch1);
3309 }
3310 }
3311 %}
3312
3313 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3314 Register src_reg = as_Register($src$$reg);
3315 Register base = as_Register($mem$$base);
3316 int index = $mem$$index;
3317 int scale = $mem$$scale;
3318 int disp = $mem$$disp;
3319 if (index == -1) {
3320 if (disp != 0) {
3321 __ lea(rscratch2, Address(base, disp));
3322 __ stlxr(rscratch1, src_reg, rscratch2);
3323 } else {
3324 // TODO
3325 // should we ever get anything other than this case?
3326 __ stlxr(rscratch1, src_reg, base);
3327 }
3328 } else {
3329 Register index_reg = as_Register(index);
3330 if (disp == 0) {
3331 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3332 __ stlxr(rscratch1, src_reg, rscratch2);
3333 } else {
3334 __ lea(rscratch2, Address(base, disp));
3335 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3336 __ stlxr(rscratch1, src_reg, rscratch2);
3337 }
3338 }
3339 __ cmpw(rscratch1, zr);
3340 %}
3341
3342 // prefetch encodings
3343
3344 enc_class aarch64_enc_prefetchw(memory mem) %{
3345 Register base = as_Register($mem$$base);
3346 int index = $mem$$index;
3347 int scale = $mem$$scale;
3348 int disp = $mem$$disp;
3349 if (index == -1) {
3350 // Fix up any out-of-range offsets.
3351 assert_different_registers(rscratch1, base);
3352 Address addr = Address(base, disp);
3353 addr = __ legitimize_address(addr, 8, rscratch1);
3354 __ prfm(addr, PSTL1KEEP);
3355 } else {
3356 Register index_reg = as_Register(index);
3357 if (disp == 0) {
3358 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3359 } else {
3360 __ lea(rscratch1, Address(base, disp));
3361 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3362 }
3363 }
3364 %}
3365
3366 // mov encodings
3367
3368 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3369 uint32_t con = (uint32_t)$src$$constant;
3370 Register dst_reg = as_Register($dst$$reg);
3371 if (con == 0) {
3372 __ movw(dst_reg, zr);
3373 } else {
3374 __ movw(dst_reg, con);
3375 }
3376 %}
3377
3378 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3379 Register dst_reg = as_Register($dst$$reg);
3380 uint64_t con = (uint64_t)$src$$constant;
3381 if (con == 0) {
3382 __ mov(dst_reg, zr);
3383 } else {
3384 __ mov(dst_reg, con);
3385 }
3386 %}
3387
3388 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3389 Register dst_reg = as_Register($dst$$reg);
3390 address con = (address)$src$$constant;
3391 if (con == nullptr || con == (address)1) {
3392 ShouldNotReachHere();
3393 } else {
3394 relocInfo::relocType rtype = $src->constant_reloc();
3395 if (rtype == relocInfo::oop_type) {
3396 __ movoop(dst_reg, (jobject)con);
3397 } else if (rtype == relocInfo::metadata_type) {
3398 __ mov_metadata(dst_reg, (Metadata*)con);
3399 } else {
3400 assert(rtype == relocInfo::none, "unexpected reloc type");
3401 if (! __ is_valid_AArch64_address(con) ||
3402 con < (address)(uintptr_t)os::vm_page_size()) {
3403 __ mov(dst_reg, con);
3404 } else {
3405 uint64_t offset;
3406 __ adrp(dst_reg, con, offset);
3407 __ add(dst_reg, dst_reg, offset);
3408 }
3409 }
3410 }
3411 %}
3412
3413 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3414 Register dst_reg = as_Register($dst$$reg);
3415 __ mov(dst_reg, zr);
3416 %}
3417
3418 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3419 Register dst_reg = as_Register($dst$$reg);
3420 __ mov(dst_reg, (uint64_t)1);
3421 %}
3422
3423 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3424 Register dst_reg = as_Register($dst$$reg);
3425 address con = (address)$src$$constant;
3426 if (con == nullptr) {
3427 ShouldNotReachHere();
3428 } else {
3429 relocInfo::relocType rtype = $src->constant_reloc();
3430 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3431 __ set_narrow_oop(dst_reg, (jobject)con);
3432 }
3433 %}
3434
3435 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3436 Register dst_reg = as_Register($dst$$reg);
3437 __ mov(dst_reg, zr);
3438 %}
3439
3440 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3441 Register dst_reg = as_Register($dst$$reg);
3442 address con = (address)$src$$constant;
3443 if (con == nullptr) {
3444 ShouldNotReachHere();
3445 } else {
3446 relocInfo::relocType rtype = $src->constant_reloc();
3447 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3448 __ set_narrow_klass(dst_reg, (Klass *)con);
3449 }
3450 %}
3451
3452 // arithmetic encodings
3453
3454 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3455 Register dst_reg = as_Register($dst$$reg);
3456 Register src_reg = as_Register($src1$$reg);
3457 int32_t con = (int32_t)$src2$$constant;
3458 // add has primary == 0, subtract has primary == 1
3459 if ($primary) { con = -con; }
3460 if (con < 0) {
3461 __ subw(dst_reg, src_reg, -con);
3462 } else {
3463 __ addw(dst_reg, src_reg, con);
3464 }
3465 %}
3466
3467 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3468 Register dst_reg = as_Register($dst$$reg);
3469 Register src_reg = as_Register($src1$$reg);
3470 int32_t con = (int32_t)$src2$$constant;
3471 // add has primary == 0, subtract has primary == 1
3472 if ($primary) { con = -con; }
3473 if (con < 0) {
3474 __ sub(dst_reg, src_reg, -con);
3475 } else {
3476 __ add(dst_reg, src_reg, con);
3477 }
3478 %}
3479
3480 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3481 Register dst_reg = as_Register($dst$$reg);
3482 Register src1_reg = as_Register($src1$$reg);
3483 Register src2_reg = as_Register($src2$$reg);
3484 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3485 %}
3486
3487 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3488 Register dst_reg = as_Register($dst$$reg);
3489 Register src1_reg = as_Register($src1$$reg);
3490 Register src2_reg = as_Register($src2$$reg);
3491 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3492 %}
3493
3494 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3495 Register dst_reg = as_Register($dst$$reg);
3496 Register src1_reg = as_Register($src1$$reg);
3497 Register src2_reg = as_Register($src2$$reg);
3498 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3499 %}
3500
3501 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3502 Register dst_reg = as_Register($dst$$reg);
3503 Register src1_reg = as_Register($src1$$reg);
3504 Register src2_reg = as_Register($src2$$reg);
3505 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3506 %}
3507
3508 // compare instruction encodings
3509
3510 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3511 Register reg1 = as_Register($src1$$reg);
3512 Register reg2 = as_Register($src2$$reg);
3513 __ cmpw(reg1, reg2);
3514 %}
3515
3516 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3517 Register reg = as_Register($src1$$reg);
3518 int32_t val = $src2$$constant;
3519 if (val >= 0) {
3520 __ subsw(zr, reg, val);
3521 } else {
3522 __ addsw(zr, reg, -val);
3523 }
3524 %}
3525
3526 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3527 Register reg1 = as_Register($src1$$reg);
3528 uint32_t val = (uint32_t)$src2$$constant;
3529 __ movw(rscratch1, val);
3530 __ cmpw(reg1, rscratch1);
3531 %}
3532
3533 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3534 Register reg1 = as_Register($src1$$reg);
3535 Register reg2 = as_Register($src2$$reg);
3536 __ cmp(reg1, reg2);
3537 %}
3538
3539 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3540 Register reg = as_Register($src1$$reg);
3541 int64_t val = $src2$$constant;
3542 if (val >= 0) {
3543 __ subs(zr, reg, val);
3544 } else if (val != -val) {
3545 __ adds(zr, reg, -val);
3546 } else {
3547 // aargh, Long.MIN_VALUE is a special case
3548 __ orr(rscratch1, zr, (uint64_t)val);
3549 __ subs(zr, reg, rscratch1);
3550 }
3551 %}
3552
3553 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3554 Register reg1 = as_Register($src1$$reg);
3555 uint64_t val = (uint64_t)$src2$$constant;
3556 __ mov(rscratch1, val);
3557 __ cmp(reg1, rscratch1);
3558 %}
3559
3560 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3561 Register reg1 = as_Register($src1$$reg);
3562 Register reg2 = as_Register($src2$$reg);
3563 __ cmp(reg1, reg2);
3564 %}
3565
3566 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3567 Register reg1 = as_Register($src1$$reg);
3568 Register reg2 = as_Register($src2$$reg);
3569 __ cmpw(reg1, reg2);
3570 %}
3571
3572 enc_class aarch64_enc_testp(iRegP src) %{
3573 Register reg = as_Register($src$$reg);
3574 __ cmp(reg, zr);
3575 %}
3576
3577 enc_class aarch64_enc_testn(iRegN src) %{
3578 Register reg = as_Register($src$$reg);
3579 __ cmpw(reg, zr);
3580 %}
3581
3582 enc_class aarch64_enc_b(label lbl) %{
3583 Label *L = $lbl$$label;
3584 __ b(*L);
3585 %}
3586
3587 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3588 Label *L = $lbl$$label;
3589 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3590 %}
3591
3592 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3593 Label *L = $lbl$$label;
3594 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3595 %}
3596
3597 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3598 %{
3599 Register sub_reg = as_Register($sub$$reg);
3600 Register super_reg = as_Register($super$$reg);
3601 Register temp_reg = as_Register($temp$$reg);
3602 Register result_reg = as_Register($result$$reg);
3603
3604 Label miss;
3605 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3606 nullptr, &miss,
3607 /*set_cond_codes:*/ true);
3608 if ($primary) {
3609 __ mov(result_reg, zr);
3610 }
3611 __ bind(miss);
3612 %}
3613
3614 enc_class aarch64_enc_java_static_call(method meth) %{
3615 address addr = (address)$meth$$method;
3616 address call;
3617 if (!_method) {
3618 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3619 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3620 if (call == nullptr) {
3621 ciEnv::current()->record_failure("CodeCache is full");
3622 return;
3623 }
3624 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3625 // The NOP here is purely to ensure that eliding a call to
3626 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3627 __ nop();
3628 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3629 } else {
3630 int method_index = resolved_method_index(masm);
3631 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3632 : static_call_Relocation::spec(method_index);
3633 call = __ trampoline_call(Address(addr, rspec));
3634 if (call == nullptr) {
3635 ciEnv::current()->record_failure("CodeCache is full");
3636 return;
3637 }
3638 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3639 // Calls of the same statically bound method can share
3640 // a stub to the interpreter.
3641 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3642 } else {
3643 // Emit stub for static call
3644 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3645 if (stub == nullptr) {
3646 ciEnv::current()->record_failure("CodeCache is full");
3647 return;
3648 }
3649 }
3650 }
3651
3652 __ post_call_nop();
3653
3654 // Only non uncommon_trap calls need to reinitialize ptrue.
3655 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3656 __ reinitialize_ptrue();
3657 }
3658 %}
3659
3660 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3661 int method_index = resolved_method_index(masm);
3662 address call = __ ic_call((address)$meth$$method, method_index);
3663 if (call == nullptr) {
3664 ciEnv::current()->record_failure("CodeCache is full");
3665 return;
3666 }
3667 __ post_call_nop();
3668 if (Compile::current()->max_vector_size() > 0) {
3669 __ reinitialize_ptrue();
3670 }
3671 %}
3672
3673 enc_class aarch64_enc_call_epilog() %{
3674 if (VerifyStackAtCalls) {
3675 // Check that stack depth is unchanged: find majik cookie on stack
3676 __ call_Unimplemented();
3677 }
3678 %}
3679
3680 enc_class aarch64_enc_java_to_runtime(method meth) %{
3681 // some calls to generated routines (arraycopy code) are scheduled
3682 // by C2 as runtime calls. if so we can call them using a br (they
3683 // will be in a reachable segment) otherwise we have to use a blr
3684 // which loads the absolute address into a register.
3685 address entry = (address)$meth$$method;
3686 CodeBlob *cb = CodeCache::find_blob(entry);
3687 if (cb) {
3688 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3689 if (call == nullptr) {
3690 ciEnv::current()->record_failure("CodeCache is full");
3691 return;
3692 }
3693 __ post_call_nop();
3694 } else {
3695 Label retaddr;
3696 // Make the anchor frame walkable
3697 __ adr(rscratch2, retaddr);
3698 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3699 __ lea(rscratch1, RuntimeAddress(entry));
3700 __ blr(rscratch1);
3701 __ bind(retaddr);
3702 __ post_call_nop();
3703 }
3704 if (Compile::current()->max_vector_size() > 0) {
3705 __ reinitialize_ptrue();
3706 }
3707 %}
3708
3709 enc_class aarch64_enc_rethrow() %{
3710 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3711 %}
3712
3713 enc_class aarch64_enc_ret() %{
3714 #ifdef ASSERT
3715 if (Compile::current()->max_vector_size() > 0) {
3716 __ verify_ptrue();
3717 }
3718 #endif
3719 __ ret(lr);
3720 %}
3721
3722 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3723 Register target_reg = as_Register($jump_target$$reg);
3724 __ br(target_reg);
3725 %}
3726
3727 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3728 Register target_reg = as_Register($jump_target$$reg);
3729 // exception oop should be in r0
3730 // ret addr has been popped into lr
3731 // callee expects it in r3
3732 __ mov(r3, lr);
3733 __ br(target_reg);
3734 %}
3735
3736 %}
3737
3738 //----------FRAME--------------------------------------------------------------
3739 // Definition of frame structure and management information.
3740 //
3741 // S T A C K L A Y O U T Allocators stack-slot number
3742 // | (to get allocators register number
3743 // G Owned by | | v add OptoReg::stack0())
3744 // r CALLER | |
3745 // o | +--------+ pad to even-align allocators stack-slot
3746 // w V | pad0 | numbers; owned by CALLER
3747 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3748 // h ^ | in | 5
3749 // | | args | 4 Holes in incoming args owned by SELF
3750 // | | | | 3
3751 // | | +--------+
3752 // V | | old out| Empty on Intel, window on Sparc
3753 // | old |preserve| Must be even aligned.
3754 // | SP-+--------+----> Matcher::_old_SP, even aligned
3755 // | | in | 3 area for Intel ret address
3756 // Owned by |preserve| Empty on Sparc.
3757 // SELF +--------+
3758 // | | pad2 | 2 pad to align old SP
3759 // | +--------+ 1
3760 // | | locks | 0
3761 // | +--------+----> OptoReg::stack0(), even aligned
3762 // | | pad1 | 11 pad to align new SP
3763 // | +--------+
3764 // | | | 10
3765 // | | spills | 9 spills
3766 // V | | 8 (pad0 slot for callee)
3767 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3768 // ^ | out | 7
3769 // | | args | 6 Holes in outgoing args owned by CALLEE
3770 // Owned by +--------+
3771 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3772 // | new |preserve| Must be even-aligned.
3773 // | SP-+--------+----> Matcher::_new_SP, even aligned
3774 // | | |
3775 //
3776 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3777 // known from SELF's arguments and the Java calling convention.
3778 // Region 6-7 is determined per call site.
3779 // Note 2: If the calling convention leaves holes in the incoming argument
3780 // area, those holes are owned by SELF. Holes in the outgoing area
3781 // are owned by the CALLEE. Holes should not be necessary in the
3782 // incoming area, as the Java calling convention is completely under
3783 // the control of the AD file. Doubles can be sorted and packed to
3784 // avoid holes. Holes in the outgoing arguments may be necessary for
3785 // varargs C calling conventions.
3786 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3787 // even aligned with pad0 as needed.
3788 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3789 // (the latter is true on Intel but is it false on AArch64?)
3790 // region 6-11 is even aligned; it may be padded out more so that
3791 // the region from SP to FP meets the minimum stack alignment.
3792 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3793 // alignment. Region 11, pad1, may be dynamically extended so that
3794 // SP meets the minimum alignment.
3795
3796 frame %{
3797 // These three registers define part of the calling convention
3798 // between compiled code and the interpreter.
3799
3800 // Inline Cache Register or Method for I2C.
3801 inline_cache_reg(R12);
3802
3803 // Number of stack slots consumed by locking an object
3804 sync_stack_slots(2);
3805
3806 // Compiled code's Frame Pointer
3807 frame_pointer(R31);
3808
3809 // Interpreter stores its frame pointer in a register which is
3810 // stored to the stack by I2CAdaptors.
3811 // I2CAdaptors convert from interpreted java to compiled java.
3812 interpreter_frame_pointer(R29);
3813
3814 // Stack alignment requirement
3815 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3816
3817 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3818 // for calls to C. Supports the var-args backing area for register parms.
3819 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3820
3821 // The after-PROLOG location of the return address. Location of
3822 // return address specifies a type (REG or STACK) and a number
3823 // representing the register number (i.e. - use a register name) or
3824 // stack slot.
3825 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3826 // Otherwise, it is above the locks and verification slot and alignment word
3827 // TODO this may well be correct but need to check why that - 2 is there
3828 // ppc port uses 0 but we definitely need to allow for fixed_slots
3829 // which folds in the space used for monitors
3830 return_addr(STACK - 2 +
3831 align_up((Compile::current()->in_preserve_stack_slots() +
3832 Compile::current()->fixed_slots()),
3833 stack_alignment_in_slots()));
3834
3835 // Location of compiled Java return values. Same as C for now.
3836 return_value
3837 %{
3838 // TODO do we allow ideal_reg == Op_RegN???
3839 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3840 "only return normal values");
3841
3842 static const int lo[Op_RegL + 1] = { // enum name
3843 0, // Op_Node
3844 0, // Op_Set
3845 R0_num, // Op_RegN
3846 R0_num, // Op_RegI
3847 R0_num, // Op_RegP
3848 V0_num, // Op_RegF
3849 V0_num, // Op_RegD
3850 R0_num // Op_RegL
3851 };
3852
3853 static const int hi[Op_RegL + 1] = { // enum name
3854 0, // Op_Node
3855 0, // Op_Set
3856 OptoReg::Bad, // Op_RegN
3857 OptoReg::Bad, // Op_RegI
3858 R0_H_num, // Op_RegP
3859 OptoReg::Bad, // Op_RegF
3860 V0_H_num, // Op_RegD
3861 R0_H_num // Op_RegL
3862 };
3863
3864 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3865 %}
3866 %}
3867
3868 //----------ATTRIBUTES---------------------------------------------------------
3869 //----------Operand Attributes-------------------------------------------------
3870 op_attrib op_cost(1); // Required cost attribute
3871
3872 //----------Instruction Attributes---------------------------------------------
3873 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3874 ins_attrib ins_size(32); // Required size attribute (in bits)
3875 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3876 // a non-matching short branch variant
3877 // of some long branch?
3878 ins_attrib ins_alignment(4); // Required alignment attribute (must
3879 // be a power of 2) specifies the
3880 // alignment that some part of the
3881 // instruction (not necessarily the
3882 // start) requires. If > 1, a
3883 // compute_padding() function must be
3884 // provided for the instruction
3885
3886 // Whether this node is expanded during code emission into a sequence of
3887 // instructions and the first instruction can perform an implicit null check.
3888 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3889
3890 //----------OPERANDS-----------------------------------------------------------
3891 // Operand definitions must precede instruction definitions for correct parsing
3892 // in the ADLC because operands constitute user defined types which are used in
3893 // instruction definitions.
3894
3895 //----------Simple Operands----------------------------------------------------
3896
3897 // Integer operands 32 bit
3898 // 32 bit immediate
3899 operand immI()
3900 %{
3901 match(ConI);
3902
3903 op_cost(0);
3904 format %{ %}
3905 interface(CONST_INTER);
3906 %}
3907
3908 // 32 bit zero
3909 operand immI0()
3910 %{
3911 predicate(n->get_int() == 0);
3912 match(ConI);
3913
3914 op_cost(0);
3915 format %{ %}
3916 interface(CONST_INTER);
3917 %}
3918
3919 // 32 bit unit increment
3920 operand immI_1()
3921 %{
3922 predicate(n->get_int() == 1);
3923 match(ConI);
3924
3925 op_cost(0);
3926 format %{ %}
3927 interface(CONST_INTER);
3928 %}
3929
3930 // 32 bit unit decrement
3931 operand immI_M1()
3932 %{
3933 predicate(n->get_int() == -1);
3934 match(ConI);
3935
3936 op_cost(0);
3937 format %{ %}
3938 interface(CONST_INTER);
3939 %}
3940
3941 // Shift values for add/sub extension shift
3942 operand immIExt()
3943 %{
3944 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3945 match(ConI);
3946
3947 op_cost(0);
3948 format %{ %}
3949 interface(CONST_INTER);
3950 %}
3951
3952 operand immI_gt_1()
3953 %{
3954 predicate(n->get_int() > 1);
3955 match(ConI);
3956
3957 op_cost(0);
3958 format %{ %}
3959 interface(CONST_INTER);
3960 %}
3961
3962 operand immI_le_4()
3963 %{
3964 predicate(n->get_int() <= 4);
3965 match(ConI);
3966
3967 op_cost(0);
3968 format %{ %}
3969 interface(CONST_INTER);
3970 %}
3971
3972 operand immI_16()
3973 %{
3974 predicate(n->get_int() == 16);
3975 match(ConI);
3976
3977 op_cost(0);
3978 format %{ %}
3979 interface(CONST_INTER);
3980 %}
3981
3982 operand immI_24()
3983 %{
3984 predicate(n->get_int() == 24);
3985 match(ConI);
3986
3987 op_cost(0);
3988 format %{ %}
3989 interface(CONST_INTER);
3990 %}
3991
3992 operand immI_32()
3993 %{
3994 predicate(n->get_int() == 32);
3995 match(ConI);
3996
3997 op_cost(0);
3998 format %{ %}
3999 interface(CONST_INTER);
4000 %}
4001
4002 operand immI_48()
4003 %{
4004 predicate(n->get_int() == 48);
4005 match(ConI);
4006
4007 op_cost(0);
4008 format %{ %}
4009 interface(CONST_INTER);
4010 %}
4011
4012 operand immI_56()
4013 %{
4014 predicate(n->get_int() == 56);
4015 match(ConI);
4016
4017 op_cost(0);
4018 format %{ %}
4019 interface(CONST_INTER);
4020 %}
4021
4022 operand immI_255()
4023 %{
4024 predicate(n->get_int() == 255);
4025 match(ConI);
4026
4027 op_cost(0);
4028 format %{ %}
4029 interface(CONST_INTER);
4030 %}
4031
4032 operand immI_65535()
4033 %{
4034 predicate(n->get_int() == 65535);
4035 match(ConI);
4036
4037 op_cost(0);
4038 format %{ %}
4039 interface(CONST_INTER);
4040 %}
4041
4042 operand immI_positive()
4043 %{
4044 predicate(n->get_int() > 0);
4045 match(ConI);
4046
4047 op_cost(0);
4048 format %{ %}
4049 interface(CONST_INTER);
4050 %}
4051
4052 // BoolTest condition for signed compare
4053 operand immI_cmp_cond()
4054 %{
4055 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4056 match(ConI);
4057
4058 op_cost(0);
4059 format %{ %}
4060 interface(CONST_INTER);
4061 %}
4062
4063 // BoolTest condition for unsigned compare
4064 operand immI_cmpU_cond()
4065 %{
4066 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4067 match(ConI);
4068
4069 op_cost(0);
4070 format %{ %}
4071 interface(CONST_INTER);
4072 %}
4073
4074 operand immL_255()
4075 %{
4076 predicate(n->get_long() == 255L);
4077 match(ConL);
4078
4079 op_cost(0);
4080 format %{ %}
4081 interface(CONST_INTER);
4082 %}
4083
4084 operand immL_65535()
4085 %{
4086 predicate(n->get_long() == 65535L);
4087 match(ConL);
4088
4089 op_cost(0);
4090 format %{ %}
4091 interface(CONST_INTER);
4092 %}
4093
4094 operand immL_4294967295()
4095 %{
4096 predicate(n->get_long() == 4294967295L);
4097 match(ConL);
4098
4099 op_cost(0);
4100 format %{ %}
4101 interface(CONST_INTER);
4102 %}
4103
4104 operand immL_bitmask()
4105 %{
4106 predicate((n->get_long() != 0)
4107 && ((n->get_long() & 0xc000000000000000l) == 0)
4108 && is_power_of_2(n->get_long() + 1));
4109 match(ConL);
4110
4111 op_cost(0);
4112 format %{ %}
4113 interface(CONST_INTER);
4114 %}
4115
4116 operand immI_bitmask()
4117 %{
4118 predicate((n->get_int() != 0)
4119 && ((n->get_int() & 0xc0000000) == 0)
4120 && is_power_of_2(n->get_int() + 1));
4121 match(ConI);
4122
4123 op_cost(0);
4124 format %{ %}
4125 interface(CONST_INTER);
4126 %}
4127
4128 operand immL_positive_bitmaskI()
4129 %{
4130 predicate((n->get_long() != 0)
4131 && ((julong)n->get_long() < 0x80000000ULL)
4132 && is_power_of_2(n->get_long() + 1));
4133 match(ConL);
4134
4135 op_cost(0);
4136 format %{ %}
4137 interface(CONST_INTER);
4138 %}
4139
4140 // Scale values for scaled offset addressing modes (up to long but not quad)
4141 operand immIScale()
4142 %{
4143 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4144 match(ConI);
4145
4146 op_cost(0);
4147 format %{ %}
4148 interface(CONST_INTER);
4149 %}
4150
4151 // 5 bit signed integer
4152 operand immI5()
4153 %{
4154 predicate(Assembler::is_simm(n->get_int(), 5));
4155 match(ConI);
4156
4157 op_cost(0);
4158 format %{ %}
4159 interface(CONST_INTER);
4160 %}
4161
4162 // 7 bit unsigned integer
4163 operand immIU7()
4164 %{
4165 predicate(Assembler::is_uimm(n->get_int(), 7));
4166 match(ConI);
4167
4168 op_cost(0);
4169 format %{ %}
4170 interface(CONST_INTER);
4171 %}
4172
4173 // Offset for scaled or unscaled immediate loads and stores
4174 operand immIOffset()
4175 %{
4176 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4177 match(ConI);
4178
4179 op_cost(0);
4180 format %{ %}
4181 interface(CONST_INTER);
4182 %}
4183
4184 operand immIOffset1()
4185 %{
4186 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4187 match(ConI);
4188
4189 op_cost(0);
4190 format %{ %}
4191 interface(CONST_INTER);
4192 %}
4193
4194 operand immIOffset2()
4195 %{
4196 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4197 match(ConI);
4198
4199 op_cost(0);
4200 format %{ %}
4201 interface(CONST_INTER);
4202 %}
4203
4204 operand immIOffset4()
4205 %{
4206 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4207 match(ConI);
4208
4209 op_cost(0);
4210 format %{ %}
4211 interface(CONST_INTER);
4212 %}
4213
4214 operand immIOffset8()
4215 %{
4216 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4217 match(ConI);
4218
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 operand immIOffset16()
4225 %{
4226 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4227 match(ConI);
4228
4229 op_cost(0);
4230 format %{ %}
4231 interface(CONST_INTER);
4232 %}
4233
4234 operand immLOffset()
4235 %{
4236 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4237 match(ConL);
4238
4239 op_cost(0);
4240 format %{ %}
4241 interface(CONST_INTER);
4242 %}
4243
4244 operand immLoffset1()
4245 %{
4246 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4247 match(ConL);
4248
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 operand immLoffset2()
4255 %{
4256 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4257 match(ConL);
4258
4259 op_cost(0);
4260 format %{ %}
4261 interface(CONST_INTER);
4262 %}
4263
4264 operand immLoffset4()
4265 %{
4266 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4267 match(ConL);
4268
4269 op_cost(0);
4270 format %{ %}
4271 interface(CONST_INTER);
4272 %}
4273
4274 operand immLoffset8()
4275 %{
4276 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4277 match(ConL);
4278
4279 op_cost(0);
4280 format %{ %}
4281 interface(CONST_INTER);
4282 %}
4283
4284 operand immLoffset16()
4285 %{
4286 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4287 match(ConL);
4288
4289 op_cost(0);
4290 format %{ %}
4291 interface(CONST_INTER);
4292 %}
4293
4294 // 5 bit signed long integer
4295 operand immL5()
4296 %{
4297 predicate(Assembler::is_simm(n->get_long(), 5));
4298 match(ConL);
4299
4300 op_cost(0);
4301 format %{ %}
4302 interface(CONST_INTER);
4303 %}
4304
4305 // 7 bit unsigned long integer
4306 operand immLU7()
4307 %{
4308 predicate(Assembler::is_uimm(n->get_long(), 7));
4309 match(ConL);
4310
4311 op_cost(0);
4312 format %{ %}
4313 interface(CONST_INTER);
4314 %}
4315
4316 // 8 bit signed value.
4317 operand immI8()
4318 %{
4319 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4320 match(ConI);
4321
4322 op_cost(0);
4323 format %{ %}
4324 interface(CONST_INTER);
4325 %}
4326
4327 // 8 bit signed value (simm8), or #simm8 LSL 8.
4328 operand immIDupV()
4329 %{
4330 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4331 match(ConI);
4332
4333 op_cost(0);
4334 format %{ %}
4335 interface(CONST_INTER);
4336 %}
4337
4338 // 8 bit signed value (simm8), or #simm8 LSL 8.
4339 operand immLDupV()
4340 %{
4341 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4342 match(ConL);
4343
4344 op_cost(0);
4345 format %{ %}
4346 interface(CONST_INTER);
4347 %}
4348
4349 // 8 bit signed value (simm8), or #simm8 LSL 8.
4350 operand immHDupV()
4351 %{
4352 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4353 match(ConH);
4354
4355 op_cost(0);
4356 format %{ %}
4357 interface(CONST_INTER);
4358 %}
4359
4360 // 8 bit integer valid for vector add sub immediate
4361 operand immBAddSubV()
4362 %{
4363 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4364 match(ConI);
4365
4366 op_cost(0);
4367 format %{ %}
4368 interface(CONST_INTER);
4369 %}
4370
4371 // 32 bit integer valid for add sub immediate
4372 operand immIAddSub()
4373 %{
4374 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4375 match(ConI);
4376 op_cost(0);
4377 format %{ %}
4378 interface(CONST_INTER);
4379 %}
4380
4381 // 32 bit integer valid for vector add sub immediate
4382 operand immIAddSubV()
4383 %{
4384 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4385 match(ConI);
4386
4387 op_cost(0);
4388 format %{ %}
4389 interface(CONST_INTER);
4390 %}
4391
4392 // 32 bit unsigned integer valid for logical immediate
4393
4394 operand immBLog()
4395 %{
4396 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4397 match(ConI);
4398
4399 op_cost(0);
4400 format %{ %}
4401 interface(CONST_INTER);
4402 %}
4403
4404 operand immSLog()
4405 %{
4406 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4407 match(ConI);
4408
4409 op_cost(0);
4410 format %{ %}
4411 interface(CONST_INTER);
4412 %}
4413
4414 operand immILog()
4415 %{
4416 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4417 match(ConI);
4418
4419 op_cost(0);
4420 format %{ %}
4421 interface(CONST_INTER);
4422 %}
4423
4424 // Integer operands 64 bit
4425 // 64 bit immediate
4426 operand immL()
4427 %{
4428 match(ConL);
4429
4430 op_cost(0);
4431 format %{ %}
4432 interface(CONST_INTER);
4433 %}
4434
4435 // 64 bit zero
4436 operand immL0()
4437 %{
4438 predicate(n->get_long() == 0);
4439 match(ConL);
4440
4441 op_cost(0);
4442 format %{ %}
4443 interface(CONST_INTER);
4444 %}
4445
4446 // 64 bit unit decrement
4447 operand immL_M1()
4448 %{
4449 predicate(n->get_long() == -1);
4450 match(ConL);
4451
4452 op_cost(0);
4453 format %{ %}
4454 interface(CONST_INTER);
4455 %}
4456
4457 // 64 bit integer valid for add sub immediate
4458 operand immLAddSub()
4459 %{
4460 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4461 match(ConL);
4462 op_cost(0);
4463 format %{ %}
4464 interface(CONST_INTER);
4465 %}
4466
4467 // 64 bit integer valid for addv subv immediate
4468 operand immLAddSubV()
4469 %{
4470 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4471 match(ConL);
4472
4473 op_cost(0);
4474 format %{ %}
4475 interface(CONST_INTER);
4476 %}
4477
4478 // 64 bit integer valid for logical immediate
4479 operand immLLog()
4480 %{
4481 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4482 match(ConL);
4483 op_cost(0);
4484 format %{ %}
4485 interface(CONST_INTER);
4486 %}
4487
4488 // Long Immediate: low 32-bit mask
4489 operand immL_32bits()
4490 %{
4491 predicate(n->get_long() == 0xFFFFFFFFL);
4492 match(ConL);
4493 op_cost(0);
4494 format %{ %}
4495 interface(CONST_INTER);
4496 %}
4497
4498 // Pointer operands
4499 // Pointer Immediate
4500 operand immP()
4501 %{
4502 match(ConP);
4503
4504 op_cost(0);
4505 format %{ %}
4506 interface(CONST_INTER);
4507 %}
4508
4509 // nullptr Pointer Immediate
4510 operand immP0()
4511 %{
4512 predicate(n->get_ptr() == 0);
4513 match(ConP);
4514
4515 op_cost(0);
4516 format %{ %}
4517 interface(CONST_INTER);
4518 %}
4519
4520 // Pointer Immediate One
4521 // this is used in object initialization (initial object header)
4522 operand immP_1()
4523 %{
4524 predicate(n->get_ptr() == 1);
4525 match(ConP);
4526
4527 op_cost(0);
4528 format %{ %}
4529 interface(CONST_INTER);
4530 %}
4531
4532 // Float and Double operands
4533 // Double Immediate
4534 operand immD()
4535 %{
4536 match(ConD);
4537 op_cost(0);
4538 format %{ %}
4539 interface(CONST_INTER);
4540 %}
4541
4542 // Double Immediate: +0.0d
4543 operand immD0()
4544 %{
4545 predicate(jlong_cast(n->getd()) == 0);
4546 match(ConD);
4547
4548 op_cost(0);
4549 format %{ %}
4550 interface(CONST_INTER);
4551 %}
4552
4553 // constant 'double +0.0'.
4554 operand immDPacked()
4555 %{
4556 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4557 match(ConD);
4558 op_cost(0);
4559 format %{ %}
4560 interface(CONST_INTER);
4561 %}
4562
4563 // Float Immediate
4564 operand immF()
4565 %{
4566 match(ConF);
4567 op_cost(0);
4568 format %{ %}
4569 interface(CONST_INTER);
4570 %}
4571
4572 // Float Immediate: +0.0f.
4573 operand immF0()
4574 %{
4575 predicate(jint_cast(n->getf()) == 0);
4576 match(ConF);
4577
4578 op_cost(0);
4579 format %{ %}
4580 interface(CONST_INTER);
4581 %}
4582
4583 // Half Float (FP16) Immediate
4584 operand immH()
4585 %{
4586 match(ConH);
4587 op_cost(0);
4588 format %{ %}
4589 interface(CONST_INTER);
4590 %}
4591
4592 //
4593 operand immFPacked()
4594 %{
4595 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4596 match(ConF);
4597 op_cost(0);
4598 format %{ %}
4599 interface(CONST_INTER);
4600 %}
4601
4602 // Narrow pointer operands
4603 // Narrow Pointer Immediate
4604 operand immN()
4605 %{
4606 match(ConN);
4607
4608 op_cost(0);
4609 format %{ %}
4610 interface(CONST_INTER);
4611 %}
4612
4613 // Narrow nullptr Pointer Immediate
4614 operand immN0()
4615 %{
4616 predicate(n->get_narrowcon() == 0);
4617 match(ConN);
4618
4619 op_cost(0);
4620 format %{ %}
4621 interface(CONST_INTER);
4622 %}
4623
4624 operand immNKlass()
4625 %{
4626 match(ConNKlass);
4627
4628 op_cost(0);
4629 format %{ %}
4630 interface(CONST_INTER);
4631 %}
4632
4633 // Integer 32 bit Register Operands
4634 // Integer 32 bitRegister (excludes SP)
4635 operand iRegI()
4636 %{
4637 constraint(ALLOC_IN_RC(any_reg32));
4638 match(RegI);
4639 match(iRegINoSp);
4640 op_cost(0);
4641 format %{ %}
4642 interface(REG_INTER);
4643 %}
4644
4645 // Integer 32 bit Register not Special
4646 operand iRegINoSp()
4647 %{
4648 constraint(ALLOC_IN_RC(no_special_reg32));
4649 match(RegI);
4650 op_cost(0);
4651 format %{ %}
4652 interface(REG_INTER);
4653 %}
4654
4655 // Integer 64 bit Register Operands
4656 // Integer 64 bit Register (includes SP)
4657 operand iRegL()
4658 %{
4659 constraint(ALLOC_IN_RC(any_reg));
4660 match(RegL);
4661 match(iRegLNoSp);
4662 op_cost(0);
4663 format %{ %}
4664 interface(REG_INTER);
4665 %}
4666
4667 // Integer 64 bit Register not Special
4668 operand iRegLNoSp()
4669 %{
4670 constraint(ALLOC_IN_RC(no_special_reg));
4671 match(RegL);
4672 match(iRegL_R0);
4673 format %{ %}
4674 interface(REG_INTER);
4675 %}
4676
4677 // Pointer Register Operands
4678 // Pointer Register
4679 operand iRegP()
4680 %{
4681 constraint(ALLOC_IN_RC(ptr_reg));
4682 match(RegP);
4683 match(iRegPNoSp);
4684 match(iRegP_R0);
4685 //match(iRegP_R2);
4686 //match(iRegP_R4);
4687 match(iRegP_R5);
4688 match(thread_RegP);
4689 op_cost(0);
4690 format %{ %}
4691 interface(REG_INTER);
4692 %}
4693
4694 // Pointer 64 bit Register not Special
4695 operand iRegPNoSp()
4696 %{
4697 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4698 match(RegP);
4699 // match(iRegP);
4700 // match(iRegP_R0);
4701 // match(iRegP_R2);
4702 // match(iRegP_R4);
4703 // match(iRegP_R5);
4704 // match(thread_RegP);
4705 op_cost(0);
4706 format %{ %}
4707 interface(REG_INTER);
4708 %}
4709
4710 // This operand is not allowed to use rfp even if
4711 // rfp is not used to hold the frame pointer.
4712 operand iRegPNoSpNoRfp()
4713 %{
4714 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4715 match(RegP);
4716 match(iRegPNoSp);
4717 op_cost(0);
4718 format %{ %}
4719 interface(REG_INTER);
4720 %}
4721
4722 // Pointer 64 bit Register R0 only
4723 operand iRegP_R0()
4724 %{
4725 constraint(ALLOC_IN_RC(r0_reg));
4726 match(RegP);
4727 // match(iRegP);
4728 match(iRegPNoSp);
4729 op_cost(0);
4730 format %{ %}
4731 interface(REG_INTER);
4732 %}
4733
4734 // Pointer 64 bit Register R1 only
4735 operand iRegP_R1()
4736 %{
4737 constraint(ALLOC_IN_RC(r1_reg));
4738 match(RegP);
4739 // match(iRegP);
4740 match(iRegPNoSp);
4741 op_cost(0);
4742 format %{ %}
4743 interface(REG_INTER);
4744 %}
4745
4746 // Pointer 64 bit Register R2 only
4747 operand iRegP_R2()
4748 %{
4749 constraint(ALLOC_IN_RC(r2_reg));
4750 match(RegP);
4751 // match(iRegP);
4752 match(iRegPNoSp);
4753 op_cost(0);
4754 format %{ %}
4755 interface(REG_INTER);
4756 %}
4757
4758 // Pointer 64 bit Register R3 only
4759 operand iRegP_R3()
4760 %{
4761 constraint(ALLOC_IN_RC(r3_reg));
4762 match(RegP);
4763 // match(iRegP);
4764 match(iRegPNoSp);
4765 op_cost(0);
4766 format %{ %}
4767 interface(REG_INTER);
4768 %}
4769
4770 // Pointer 64 bit Register R4 only
4771 operand iRegP_R4()
4772 %{
4773 constraint(ALLOC_IN_RC(r4_reg));
4774 match(RegP);
4775 // match(iRegP);
4776 match(iRegPNoSp);
4777 op_cost(0);
4778 format %{ %}
4779 interface(REG_INTER);
4780 %}
4781
4782 // Pointer 64 bit Register R5 only
4783 operand iRegP_R5()
4784 %{
4785 constraint(ALLOC_IN_RC(r5_reg));
4786 match(RegP);
4787 // match(iRegP);
4788 match(iRegPNoSp);
4789 op_cost(0);
4790 format %{ %}
4791 interface(REG_INTER);
4792 %}
4793
4794 // Pointer 64 bit Register R10 only
4795 operand iRegP_R10()
4796 %{
4797 constraint(ALLOC_IN_RC(r10_reg));
4798 match(RegP);
4799 // match(iRegP);
4800 match(iRegPNoSp);
4801 op_cost(0);
4802 format %{ %}
4803 interface(REG_INTER);
4804 %}
4805
4806 // Long 64 bit Register R0 only
4807 operand iRegL_R0()
4808 %{
4809 constraint(ALLOC_IN_RC(r0_reg));
4810 match(RegL);
4811 match(iRegLNoSp);
4812 op_cost(0);
4813 format %{ %}
4814 interface(REG_INTER);
4815 %}
4816
4817 // Long 64 bit Register R11 only
4818 operand iRegL_R11()
4819 %{
4820 constraint(ALLOC_IN_RC(r11_reg));
4821 match(RegL);
4822 match(iRegLNoSp);
4823 op_cost(0);
4824 format %{ %}
4825 interface(REG_INTER);
4826 %}
4827
4828 // Register R0 only
4829 operand iRegI_R0()
4830 %{
4831 constraint(ALLOC_IN_RC(int_r0_reg));
4832 match(RegI);
4833 match(iRegINoSp);
4834 op_cost(0);
4835 format %{ %}
4836 interface(REG_INTER);
4837 %}
4838
4839 // Register R2 only
4840 operand iRegI_R2()
4841 %{
4842 constraint(ALLOC_IN_RC(int_r2_reg));
4843 match(RegI);
4844 match(iRegINoSp);
4845 op_cost(0);
4846 format %{ %}
4847 interface(REG_INTER);
4848 %}
4849
4850 // Register R3 only
4851 operand iRegI_R3()
4852 %{
4853 constraint(ALLOC_IN_RC(int_r3_reg));
4854 match(RegI);
4855 match(iRegINoSp);
4856 op_cost(0);
4857 format %{ %}
4858 interface(REG_INTER);
4859 %}
4860
4861
4862 // Register R4 only
4863 operand iRegI_R4()
4864 %{
4865 constraint(ALLOC_IN_RC(int_r4_reg));
4866 match(RegI);
4867 match(iRegINoSp);
4868 op_cost(0);
4869 format %{ %}
4870 interface(REG_INTER);
4871 %}
4872
4873
4874 // Pointer Register Operands
4875 // Narrow Pointer Register
4876 operand iRegN()
4877 %{
4878 constraint(ALLOC_IN_RC(any_reg32));
4879 match(RegN);
4880 match(iRegNNoSp);
4881 op_cost(0);
4882 format %{ %}
4883 interface(REG_INTER);
4884 %}
4885
4886 // Integer 64 bit Register not Special
4887 operand iRegNNoSp()
4888 %{
4889 constraint(ALLOC_IN_RC(no_special_reg32));
4890 match(RegN);
4891 op_cost(0);
4892 format %{ %}
4893 interface(REG_INTER);
4894 %}
4895
4896 // Float Register
4897 // Float register operands
4898 operand vRegF()
4899 %{
4900 constraint(ALLOC_IN_RC(float_reg));
4901 match(RegF);
4902
4903 op_cost(0);
4904 format %{ %}
4905 interface(REG_INTER);
4906 %}
4907
4908 // Double Register
4909 // Double register operands
4910 operand vRegD()
4911 %{
4912 constraint(ALLOC_IN_RC(double_reg));
4913 match(RegD);
4914
4915 op_cost(0);
4916 format %{ %}
4917 interface(REG_INTER);
4918 %}
4919
4920 // Generic vector class. This will be used for
4921 // all vector operands, including NEON and SVE.
4922 operand vReg()
4923 %{
4924 constraint(ALLOC_IN_RC(dynamic));
4925 match(VecA);
4926 match(VecD);
4927 match(VecX);
4928
4929 op_cost(0);
4930 format %{ %}
4931 interface(REG_INTER);
4932 %}
4933
4934 operand vReg_V10()
4935 %{
4936 constraint(ALLOC_IN_RC(v10_veca_reg));
4937 match(vReg);
4938
4939 op_cost(0);
4940 format %{ %}
4941 interface(REG_INTER);
4942 %}
4943
4944 operand vReg_V11()
4945 %{
4946 constraint(ALLOC_IN_RC(v11_veca_reg));
4947 match(vReg);
4948
4949 op_cost(0);
4950 format %{ %}
4951 interface(REG_INTER);
4952 %}
4953
4954 operand vReg_V12()
4955 %{
4956 constraint(ALLOC_IN_RC(v12_veca_reg));
4957 match(vReg);
4958
4959 op_cost(0);
4960 format %{ %}
4961 interface(REG_INTER);
4962 %}
4963
4964 operand vReg_V13()
4965 %{
4966 constraint(ALLOC_IN_RC(v13_veca_reg));
4967 match(vReg);
4968
4969 op_cost(0);
4970 format %{ %}
4971 interface(REG_INTER);
4972 %}
4973
4974 operand vReg_V17()
4975 %{
4976 constraint(ALLOC_IN_RC(v17_veca_reg));
4977 match(vReg);
4978
4979 op_cost(0);
4980 format %{ %}
4981 interface(REG_INTER);
4982 %}
4983
4984 operand vReg_V18()
4985 %{
4986 constraint(ALLOC_IN_RC(v18_veca_reg));
4987 match(vReg);
4988
4989 op_cost(0);
4990 format %{ %}
4991 interface(REG_INTER);
4992 %}
4993
4994 operand vReg_V23()
4995 %{
4996 constraint(ALLOC_IN_RC(v23_veca_reg));
4997 match(vReg);
4998
4999 op_cost(0);
5000 format %{ %}
5001 interface(REG_INTER);
5002 %}
5003
5004 operand vReg_V24()
5005 %{
5006 constraint(ALLOC_IN_RC(v24_veca_reg));
5007 match(vReg);
5008
5009 op_cost(0);
5010 format %{ %}
5011 interface(REG_INTER);
5012 %}
5013
5014 operand vecA()
5015 %{
5016 constraint(ALLOC_IN_RC(vectora_reg));
5017 match(VecA);
5018
5019 op_cost(0);
5020 format %{ %}
5021 interface(REG_INTER);
5022 %}
5023
5024 operand vecD()
5025 %{
5026 constraint(ALLOC_IN_RC(vectord_reg));
5027 match(VecD);
5028
5029 op_cost(0);
5030 format %{ %}
5031 interface(REG_INTER);
5032 %}
5033
5034 operand vecX()
5035 %{
5036 constraint(ALLOC_IN_RC(vectorx_reg));
5037 match(VecX);
5038
5039 op_cost(0);
5040 format %{ %}
5041 interface(REG_INTER);
5042 %}
5043
5044 operand vRegD_V0()
5045 %{
5046 constraint(ALLOC_IN_RC(v0_reg));
5047 match(RegD);
5048 op_cost(0);
5049 format %{ %}
5050 interface(REG_INTER);
5051 %}
5052
5053 operand vRegD_V1()
5054 %{
5055 constraint(ALLOC_IN_RC(v1_reg));
5056 match(RegD);
5057 op_cost(0);
5058 format %{ %}
5059 interface(REG_INTER);
5060 %}
5061
5062 operand vRegD_V2()
5063 %{
5064 constraint(ALLOC_IN_RC(v2_reg));
5065 match(RegD);
5066 op_cost(0);
5067 format %{ %}
5068 interface(REG_INTER);
5069 %}
5070
5071 operand vRegD_V3()
5072 %{
5073 constraint(ALLOC_IN_RC(v3_reg));
5074 match(RegD);
5075 op_cost(0);
5076 format %{ %}
5077 interface(REG_INTER);
5078 %}
5079
5080 operand vRegD_V4()
5081 %{
5082 constraint(ALLOC_IN_RC(v4_reg));
5083 match(RegD);
5084 op_cost(0);
5085 format %{ %}
5086 interface(REG_INTER);
5087 %}
5088
5089 operand vRegD_V5()
5090 %{
5091 constraint(ALLOC_IN_RC(v5_reg));
5092 match(RegD);
5093 op_cost(0);
5094 format %{ %}
5095 interface(REG_INTER);
5096 %}
5097
5098 operand vRegD_V6()
5099 %{
5100 constraint(ALLOC_IN_RC(v6_reg));
5101 match(RegD);
5102 op_cost(0);
5103 format %{ %}
5104 interface(REG_INTER);
5105 %}
5106
5107 operand vRegD_V7()
5108 %{
5109 constraint(ALLOC_IN_RC(v7_reg));
5110 match(RegD);
5111 op_cost(0);
5112 format %{ %}
5113 interface(REG_INTER);
5114 %}
5115
5116 operand vRegD_V12()
5117 %{
5118 constraint(ALLOC_IN_RC(v12_reg));
5119 match(RegD);
5120 op_cost(0);
5121 format %{ %}
5122 interface(REG_INTER);
5123 %}
5124
5125 operand vRegD_V13()
5126 %{
5127 constraint(ALLOC_IN_RC(v13_reg));
5128 match(RegD);
5129 op_cost(0);
5130 format %{ %}
5131 interface(REG_INTER);
5132 %}
5133
5134 operand pReg()
5135 %{
5136 constraint(ALLOC_IN_RC(pr_reg));
5137 match(RegVectMask);
5138 match(pRegGov);
5139 op_cost(0);
5140 format %{ %}
5141 interface(REG_INTER);
5142 %}
5143
5144 operand pRegGov()
5145 %{
5146 constraint(ALLOC_IN_RC(gov_pr));
5147 match(RegVectMask);
5148 match(pReg);
5149 op_cost(0);
5150 format %{ %}
5151 interface(REG_INTER);
5152 %}
5153
5154 operand pRegGov_P0()
5155 %{
5156 constraint(ALLOC_IN_RC(p0_reg));
5157 match(RegVectMask);
5158 op_cost(0);
5159 format %{ %}
5160 interface(REG_INTER);
5161 %}
5162
5163 operand pRegGov_P1()
5164 %{
5165 constraint(ALLOC_IN_RC(p1_reg));
5166 match(RegVectMask);
5167 op_cost(0);
5168 format %{ %}
5169 interface(REG_INTER);
5170 %}
5171
5172 // Flags register, used as output of signed compare instructions
5173
5174 // note that on AArch64 we also use this register as the output for
5175 // for floating point compare instructions (CmpF CmpD). this ensures
5176 // that ordered inequality tests use GT, GE, LT or LE none of which
5177 // pass through cases where the result is unordered i.e. one or both
5178 // inputs to the compare is a NaN. this means that the ideal code can
5179 // replace e.g. a GT with an LE and not end up capturing the NaN case
5180 // (where the comparison should always fail). EQ and NE tests are
5181 // always generated in ideal code so that unordered folds into the NE
5182 // case, matching the behaviour of AArch64 NE.
5183 //
5184 // This differs from x86 where the outputs of FP compares use a
5185 // special FP flags registers and where compares based on this
5186 // register are distinguished into ordered inequalities (cmpOpUCF) and
5187 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5188 // to explicitly handle the unordered case in branches. x86 also has
5189 // to include extra CMoveX rules to accept a cmpOpUCF input.
5190
5191 operand rFlagsReg()
5192 %{
5193 constraint(ALLOC_IN_RC(int_flags));
5194 match(RegFlags);
5195
5196 op_cost(0);
5197 format %{ "RFLAGS" %}
5198 interface(REG_INTER);
5199 %}
5200
5201 // Flags register, used as output of unsigned compare instructions
5202 operand rFlagsRegU()
5203 %{
5204 constraint(ALLOC_IN_RC(int_flags));
5205 match(RegFlags);
5206
5207 op_cost(0);
5208 format %{ "RFLAGSU" %}
5209 interface(REG_INTER);
5210 %}
5211
5212 // Special Registers
5213
5214 // Method Register
5215 operand inline_cache_RegP(iRegP reg)
5216 %{
5217 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5218 match(reg);
5219 match(iRegPNoSp);
5220 op_cost(0);
5221 format %{ %}
5222 interface(REG_INTER);
5223 %}
5224
5225 // Thread Register
5226 operand thread_RegP(iRegP reg)
5227 %{
5228 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5229 match(reg);
5230 op_cost(0);
5231 format %{ %}
5232 interface(REG_INTER);
5233 %}
5234
5235 //----------Memory Operands----------------------------------------------------
5236
5237 operand indirect(iRegP reg)
5238 %{
5239 constraint(ALLOC_IN_RC(ptr_reg));
5240 match(reg);
5241 op_cost(0);
5242 format %{ "[$reg]" %}
5243 interface(MEMORY_INTER) %{
5244 base($reg);
5245 index(0xffffffff);
5246 scale(0x0);
5247 disp(0x0);
5248 %}
5249 %}
5250
5251 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5252 %{
5253 constraint(ALLOC_IN_RC(ptr_reg));
5254 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5255 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5256 op_cost(0);
5257 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5258 interface(MEMORY_INTER) %{
5259 base($reg);
5260 index($ireg);
5261 scale($scale);
5262 disp(0x0);
5263 %}
5264 %}
5265
5266 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5267 %{
5268 constraint(ALLOC_IN_RC(ptr_reg));
5269 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5270 match(AddP reg (LShiftL lreg scale));
5271 op_cost(0);
5272 format %{ "$reg, $lreg lsl($scale)" %}
5273 interface(MEMORY_INTER) %{
5274 base($reg);
5275 index($lreg);
5276 scale($scale);
5277 disp(0x0);
5278 %}
5279 %}
5280
5281 operand indIndexI2L(iRegP reg, iRegI ireg)
5282 %{
5283 constraint(ALLOC_IN_RC(ptr_reg));
5284 match(AddP reg (ConvI2L ireg));
5285 op_cost(0);
5286 format %{ "$reg, $ireg, 0, I2L" %}
5287 interface(MEMORY_INTER) %{
5288 base($reg);
5289 index($ireg);
5290 scale(0x0);
5291 disp(0x0);
5292 %}
5293 %}
5294
5295 operand indIndex(iRegP reg, iRegL lreg)
5296 %{
5297 constraint(ALLOC_IN_RC(ptr_reg));
5298 match(AddP reg lreg);
5299 op_cost(0);
5300 format %{ "$reg, $lreg" %}
5301 interface(MEMORY_INTER) %{
5302 base($reg);
5303 index($lreg);
5304 scale(0x0);
5305 disp(0x0);
5306 %}
5307 %}
5308
5309 operand indOffI1(iRegP reg, immIOffset1 off)
5310 %{
5311 constraint(ALLOC_IN_RC(ptr_reg));
5312 match(AddP reg off);
5313 op_cost(0);
5314 format %{ "[$reg, $off]" %}
5315 interface(MEMORY_INTER) %{
5316 base($reg);
5317 index(0xffffffff);
5318 scale(0x0);
5319 disp($off);
5320 %}
5321 %}
5322
5323 operand indOffI2(iRegP reg, immIOffset2 off)
5324 %{
5325 constraint(ALLOC_IN_RC(ptr_reg));
5326 match(AddP reg off);
5327 op_cost(0);
5328 format %{ "[$reg, $off]" %}
5329 interface(MEMORY_INTER) %{
5330 base($reg);
5331 index(0xffffffff);
5332 scale(0x0);
5333 disp($off);
5334 %}
5335 %}
5336
5337 operand indOffI4(iRegP reg, immIOffset4 off)
5338 %{
5339 constraint(ALLOC_IN_RC(ptr_reg));
5340 match(AddP reg off);
5341 op_cost(0);
5342 format %{ "[$reg, $off]" %}
5343 interface(MEMORY_INTER) %{
5344 base($reg);
5345 index(0xffffffff);
5346 scale(0x0);
5347 disp($off);
5348 %}
5349 %}
5350
5351 operand indOffI8(iRegP reg, immIOffset8 off)
5352 %{
5353 constraint(ALLOC_IN_RC(ptr_reg));
5354 match(AddP reg off);
5355 op_cost(0);
5356 format %{ "[$reg, $off]" %}
5357 interface(MEMORY_INTER) %{
5358 base($reg);
5359 index(0xffffffff);
5360 scale(0x0);
5361 disp($off);
5362 %}
5363 %}
5364
5365 operand indOffI16(iRegP reg, immIOffset16 off)
5366 %{
5367 constraint(ALLOC_IN_RC(ptr_reg));
5368 match(AddP reg off);
5369 op_cost(0);
5370 format %{ "[$reg, $off]" %}
5371 interface(MEMORY_INTER) %{
5372 base($reg);
5373 index(0xffffffff);
5374 scale(0x0);
5375 disp($off);
5376 %}
5377 %}
5378
5379 operand indOffL1(iRegP reg, immLoffset1 off)
5380 %{
5381 constraint(ALLOC_IN_RC(ptr_reg));
5382 match(AddP reg off);
5383 op_cost(0);
5384 format %{ "[$reg, $off]" %}
5385 interface(MEMORY_INTER) %{
5386 base($reg);
5387 index(0xffffffff);
5388 scale(0x0);
5389 disp($off);
5390 %}
5391 %}
5392
5393 operand indOffL2(iRegP reg, immLoffset2 off)
5394 %{
5395 constraint(ALLOC_IN_RC(ptr_reg));
5396 match(AddP reg off);
5397 op_cost(0);
5398 format %{ "[$reg, $off]" %}
5399 interface(MEMORY_INTER) %{
5400 base($reg);
5401 index(0xffffffff);
5402 scale(0x0);
5403 disp($off);
5404 %}
5405 %}
5406
5407 operand indOffL4(iRegP reg, immLoffset4 off)
5408 %{
5409 constraint(ALLOC_IN_RC(ptr_reg));
5410 match(AddP reg off);
5411 op_cost(0);
5412 format %{ "[$reg, $off]" %}
5413 interface(MEMORY_INTER) %{
5414 base($reg);
5415 index(0xffffffff);
5416 scale(0x0);
5417 disp($off);
5418 %}
5419 %}
5420
5421 operand indOffL8(iRegP reg, immLoffset8 off)
5422 %{
5423 constraint(ALLOC_IN_RC(ptr_reg));
5424 match(AddP reg off);
5425 op_cost(0);
5426 format %{ "[$reg, $off]" %}
5427 interface(MEMORY_INTER) %{
5428 base($reg);
5429 index(0xffffffff);
5430 scale(0x0);
5431 disp($off);
5432 %}
5433 %}
5434
5435 operand indOffL16(iRegP reg, immLoffset16 off)
5436 %{
5437 constraint(ALLOC_IN_RC(ptr_reg));
5438 match(AddP reg off);
5439 op_cost(0);
5440 format %{ "[$reg, $off]" %}
5441 interface(MEMORY_INTER) %{
5442 base($reg);
5443 index(0xffffffff);
5444 scale(0x0);
5445 disp($off);
5446 %}
5447 %}
5448
5449 operand indirectX2P(iRegL reg)
5450 %{
5451 constraint(ALLOC_IN_RC(ptr_reg));
5452 match(CastX2P reg);
5453 op_cost(0);
5454 format %{ "[$reg]\t# long -> ptr" %}
5455 interface(MEMORY_INTER) %{
5456 base($reg);
5457 index(0xffffffff);
5458 scale(0x0);
5459 disp(0x0);
5460 %}
5461 %}
5462
5463 operand indOffX2P(iRegL reg, immLOffset off)
5464 %{
5465 constraint(ALLOC_IN_RC(ptr_reg));
5466 match(AddP (CastX2P reg) off);
5467 op_cost(0);
5468 format %{ "[$reg, $off]\t# long -> ptr" %}
5469 interface(MEMORY_INTER) %{
5470 base($reg);
5471 index(0xffffffff);
5472 scale(0x0);
5473 disp($off);
5474 %}
5475 %}
5476
5477 operand indirectN(iRegN reg)
5478 %{
5479 predicate(CompressedOops::shift() == 0);
5480 constraint(ALLOC_IN_RC(ptr_reg));
5481 match(DecodeN reg);
5482 op_cost(0);
5483 format %{ "[$reg]\t# narrow" %}
5484 interface(MEMORY_INTER) %{
5485 base($reg);
5486 index(0xffffffff);
5487 scale(0x0);
5488 disp(0x0);
5489 %}
5490 %}
5491
5492 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5493 %{
5494 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5495 constraint(ALLOC_IN_RC(ptr_reg));
5496 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5497 op_cost(0);
5498 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5499 interface(MEMORY_INTER) %{
5500 base($reg);
5501 index($ireg);
5502 scale($scale);
5503 disp(0x0);
5504 %}
5505 %}
5506
5507 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5508 %{
5509 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5510 constraint(ALLOC_IN_RC(ptr_reg));
5511 match(AddP (DecodeN reg) (LShiftL lreg scale));
5512 op_cost(0);
5513 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5514 interface(MEMORY_INTER) %{
5515 base($reg);
5516 index($lreg);
5517 scale($scale);
5518 disp(0x0);
5519 %}
5520 %}
5521
5522 operand indIndexI2LN(iRegN reg, iRegI ireg)
5523 %{
5524 predicate(CompressedOops::shift() == 0);
5525 constraint(ALLOC_IN_RC(ptr_reg));
5526 match(AddP (DecodeN reg) (ConvI2L ireg));
5527 op_cost(0);
5528 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5529 interface(MEMORY_INTER) %{
5530 base($reg);
5531 index($ireg);
5532 scale(0x0);
5533 disp(0x0);
5534 %}
5535 %}
5536
5537 operand indIndexN(iRegN reg, iRegL lreg)
5538 %{
5539 predicate(CompressedOops::shift() == 0);
5540 constraint(ALLOC_IN_RC(ptr_reg));
5541 match(AddP (DecodeN reg) lreg);
5542 op_cost(0);
5543 format %{ "$reg, $lreg\t# narrow" %}
5544 interface(MEMORY_INTER) %{
5545 base($reg);
5546 index($lreg);
5547 scale(0x0);
5548 disp(0x0);
5549 %}
5550 %}
5551
5552 operand indOffIN(iRegN reg, immIOffset off)
5553 %{
5554 predicate(CompressedOops::shift() == 0);
5555 constraint(ALLOC_IN_RC(ptr_reg));
5556 match(AddP (DecodeN reg) off);
5557 op_cost(0);
5558 format %{ "[$reg, $off]\t# narrow" %}
5559 interface(MEMORY_INTER) %{
5560 base($reg);
5561 index(0xffffffff);
5562 scale(0x0);
5563 disp($off);
5564 %}
5565 %}
5566
5567 operand indOffLN(iRegN reg, immLOffset off)
5568 %{
5569 predicate(CompressedOops::shift() == 0);
5570 constraint(ALLOC_IN_RC(ptr_reg));
5571 match(AddP (DecodeN reg) off);
5572 op_cost(0);
5573 format %{ "[$reg, $off]\t# narrow" %}
5574 interface(MEMORY_INTER) %{
5575 base($reg);
5576 index(0xffffffff);
5577 scale(0x0);
5578 disp($off);
5579 %}
5580 %}
5581
5582
5583 //----------Special Memory Operands--------------------------------------------
5584 // Stack Slot Operand - This operand is used for loading and storing temporary
5585 // values on the stack where a match requires a value to
5586 // flow through memory.
5587 operand stackSlotP(sRegP reg)
5588 %{
5589 constraint(ALLOC_IN_RC(stack_slots));
5590 op_cost(100);
5591 // No match rule because this operand is only generated in matching
5592 // match(RegP);
5593 format %{ "[$reg]" %}
5594 interface(MEMORY_INTER) %{
5595 base(0x1e); // RSP
5596 index(0x0); // No Index
5597 scale(0x0); // No Scale
5598 disp($reg); // Stack Offset
5599 %}
5600 %}
5601
5602 operand stackSlotI(sRegI reg)
5603 %{
5604 constraint(ALLOC_IN_RC(stack_slots));
5605 // No match rule because this operand is only generated in matching
5606 // match(RegI);
5607 format %{ "[$reg]" %}
5608 interface(MEMORY_INTER) %{
5609 base(0x1e); // RSP
5610 index(0x0); // No Index
5611 scale(0x0); // No Scale
5612 disp($reg); // Stack Offset
5613 %}
5614 %}
5615
5616 operand stackSlotF(sRegF reg)
5617 %{
5618 constraint(ALLOC_IN_RC(stack_slots));
5619 // No match rule because this operand is only generated in matching
5620 // match(RegF);
5621 format %{ "[$reg]" %}
5622 interface(MEMORY_INTER) %{
5623 base(0x1e); // RSP
5624 index(0x0); // No Index
5625 scale(0x0); // No Scale
5626 disp($reg); // Stack Offset
5627 %}
5628 %}
5629
5630 operand stackSlotD(sRegD reg)
5631 %{
5632 constraint(ALLOC_IN_RC(stack_slots));
5633 // No match rule because this operand is only generated in matching
5634 // match(RegD);
5635 format %{ "[$reg]" %}
5636 interface(MEMORY_INTER) %{
5637 base(0x1e); // RSP
5638 index(0x0); // No Index
5639 scale(0x0); // No Scale
5640 disp($reg); // Stack Offset
5641 %}
5642 %}
5643
5644 operand stackSlotL(sRegL reg)
5645 %{
5646 constraint(ALLOC_IN_RC(stack_slots));
5647 // No match rule because this operand is only generated in matching
5648 // match(RegL);
5649 format %{ "[$reg]" %}
5650 interface(MEMORY_INTER) %{
5651 base(0x1e); // RSP
5652 index(0x0); // No Index
5653 scale(0x0); // No Scale
5654 disp($reg); // Stack Offset
5655 %}
5656 %}
5657
5658 // Operands for expressing Control Flow
5659 // NOTE: Label is a predefined operand which should not be redefined in
5660 // the AD file. It is generically handled within the ADLC.
5661
5662 //----------Conditional Branch Operands----------------------------------------
5663 // Comparison Op - This is the operation of the comparison, and is limited to
5664 // the following set of codes:
5665 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5666 //
5667 // Other attributes of the comparison, such as unsignedness, are specified
5668 // by the comparison instruction that sets a condition code flags register.
5669 // That result is represented by a flags operand whose subtype is appropriate
5670 // to the unsignedness (etc.) of the comparison.
5671 //
5672 // Later, the instruction which matches both the Comparison Op (a Bool) and
5673 // the flags (produced by the Cmp) specifies the coding of the comparison op
5674 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5675
5676 // used for signed integral comparisons and fp comparisons
5677
5678 operand cmpOp()
5679 %{
5680 match(Bool);
5681
5682 format %{ "" %}
5683 interface(COND_INTER) %{
5684 equal(0x0, "eq");
5685 not_equal(0x1, "ne");
5686 less(0xb, "lt");
5687 greater_equal(0xa, "ge");
5688 less_equal(0xd, "le");
5689 greater(0xc, "gt");
5690 overflow(0x6, "vs");
5691 no_overflow(0x7, "vc");
5692 %}
5693 %}
5694
5695 // used for unsigned integral comparisons
5696
5697 operand cmpOpU()
5698 %{
5699 match(Bool);
5700
5701 format %{ "" %}
5702 interface(COND_INTER) %{
5703 equal(0x0, "eq");
5704 not_equal(0x1, "ne");
5705 less(0x3, "lo");
5706 greater_equal(0x2, "hs");
5707 less_equal(0x9, "ls");
5708 greater(0x8, "hi");
5709 overflow(0x6, "vs");
5710 no_overflow(0x7, "vc");
5711 %}
5712 %}
5713
5714 // used for certain integral comparisons which can be
5715 // converted to cbxx or tbxx instructions
5716
5717 operand cmpOpEqNe()
5718 %{
5719 match(Bool);
5720 op_cost(0);
5721 predicate(n->as_Bool()->_test._test == BoolTest::ne
5722 || n->as_Bool()->_test._test == BoolTest::eq);
5723
5724 format %{ "" %}
5725 interface(COND_INTER) %{
5726 equal(0x0, "eq");
5727 not_equal(0x1, "ne");
5728 less(0xb, "lt");
5729 greater_equal(0xa, "ge");
5730 less_equal(0xd, "le");
5731 greater(0xc, "gt");
5732 overflow(0x6, "vs");
5733 no_overflow(0x7, "vc");
5734 %}
5735 %}
5736
5737 // used for certain integral comparisons which can be
5738 // converted to cbxx or tbxx instructions
5739
5740 operand cmpOpLtGe()
5741 %{
5742 match(Bool);
5743 op_cost(0);
5744
5745 predicate(n->as_Bool()->_test._test == BoolTest::lt
5746 || n->as_Bool()->_test._test == BoolTest::ge);
5747
5748 format %{ "" %}
5749 interface(COND_INTER) %{
5750 equal(0x0, "eq");
5751 not_equal(0x1, "ne");
5752 less(0xb, "lt");
5753 greater_equal(0xa, "ge");
5754 less_equal(0xd, "le");
5755 greater(0xc, "gt");
5756 overflow(0x6, "vs");
5757 no_overflow(0x7, "vc");
5758 %}
5759 %}
5760
5761 // used for certain unsigned integral comparisons which can be
5762 // converted to cbxx or tbxx instructions
5763
5764 operand cmpOpUEqNeLeGt()
5765 %{
5766 match(Bool);
5767 op_cost(0);
5768
5769 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5770 n->as_Bool()->_test._test == BoolTest::ne ||
5771 n->as_Bool()->_test._test == BoolTest::le ||
5772 n->as_Bool()->_test._test == BoolTest::gt);
5773
5774 format %{ "" %}
5775 interface(COND_INTER) %{
5776 equal(0x0, "eq");
5777 not_equal(0x1, "ne");
5778 less(0x3, "lo");
5779 greater_equal(0x2, "hs");
5780 less_equal(0x9, "ls");
5781 greater(0x8, "hi");
5782 overflow(0x6, "vs");
5783 no_overflow(0x7, "vc");
5784 %}
5785 %}
5786
5787 // Special operand allowing long args to int ops to be truncated for free
5788
5789 operand iRegL2I(iRegL reg) %{
5790
5791 op_cost(0);
5792
5793 match(ConvL2I reg);
5794
5795 format %{ "l2i($reg)" %}
5796
5797 interface(REG_INTER)
5798 %}
5799
5800 operand iRegL2P(iRegL reg) %{
5801
5802 op_cost(0);
5803
5804 match(CastX2P reg);
5805
5806 format %{ "l2p($reg)" %}
5807
5808 interface(REG_INTER)
5809 %}
5810
5811 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5812 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5813 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5814 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5815
5816 //----------OPERAND CLASSES----------------------------------------------------
5817 // Operand Classes are groups of operands that are used as to simplify
5818 // instruction definitions by not requiring the AD writer to specify
5819 // separate instructions for every form of operand when the
5820 // instruction accepts multiple operand types with the same basic
5821 // encoding and format. The classic case of this is memory operands.
5822
5823 // memory is used to define read/write location for load/store
5824 // instruction defs. we can turn a memory op into an Address
5825
5826 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5827 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5828
5829 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5830 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5831
5832 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5833 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5834
5835 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5836 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5837
5838 // All of the memory operands. For the pipeline description.
5839 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5840 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5841 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5842
5843
5844 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5845 // operations. it allows the src to be either an iRegI or a (ConvL2I
5846 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5847 // can be elided because the 32-bit instruction will just employ the
5848 // lower 32 bits anyway.
5849 //
5850 // n.b. this does not elide all L2I conversions. if the truncated
5851 // value is consumed by more than one operation then the ConvL2I
5852 // cannot be bundled into the consuming nodes so an l2i gets planted
5853 // (actually a movw $dst $src) and the downstream instructions consume
5854 // the result of the l2i as an iRegI input. That's a shame since the
5855 // movw is actually redundant but its not too costly.
5856
5857 opclass iRegIorL2I(iRegI, iRegL2I);
5858 opclass iRegPorL2P(iRegP, iRegL2P);
5859
5860 //----------PIPELINE-----------------------------------------------------------
5861 // Rules which define the behavior of the target architectures pipeline.
5862
5863 // For specific pipelines, eg A53, define the stages of that pipeline
5864 //pipe_desc(ISS, EX1, EX2, WR);
5865 #define ISS S0
5866 #define EX1 S1
5867 #define EX2 S2
5868 #define WR S3
5869
5870 // Integer ALU reg operation
5871 pipeline %{
5872
5873 attributes %{
5874 // ARM instructions are of fixed length
5875 fixed_size_instructions; // Fixed size instructions TODO does
5876 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5877 // ARM instructions come in 32-bit word units
5878 instruction_unit_size = 4; // An instruction is 4 bytes long
5879 instruction_fetch_unit_size = 64; // The processor fetches one line
5880 instruction_fetch_units = 1; // of 64 bytes
5881 %}
5882
5883 // We don't use an actual pipeline model so don't care about resources
5884 // or description. we do use pipeline classes to introduce fixed
5885 // latencies
5886
5887 //----------RESOURCES----------------------------------------------------------
5888 // Resources are the functional units available to the machine
5889
5890 resources( INS0, INS1, INS01 = INS0 | INS1,
5891 ALU0, ALU1, ALU = ALU0 | ALU1,
5892 MAC,
5893 DIV,
5894 BRANCH,
5895 LDST,
5896 NEON_FP);
5897
5898 //----------PIPELINE DESCRIPTION-----------------------------------------------
5899 // Pipeline Description specifies the stages in the machine's pipeline
5900
5901 // Define the pipeline as a generic 6 stage pipeline
5902 pipe_desc(S0, S1, S2, S3, S4, S5);
5903
5904 //----------PIPELINE CLASSES---------------------------------------------------
5905 // Pipeline Classes describe the stages in which input and output are
5906 // referenced by the hardware pipeline.
5907
5908 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5909 %{
5910 single_instruction;
5911 src1 : S1(read);
5912 src2 : S2(read);
5913 dst : S5(write);
5914 INS01 : ISS;
5915 NEON_FP : S5;
5916 %}
5917
5918 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5919 %{
5920 single_instruction;
5921 src1 : S1(read);
5922 src2 : S2(read);
5923 dst : S5(write);
5924 INS01 : ISS;
5925 NEON_FP : S5;
5926 %}
5927
5928 pipe_class fp_uop_s(vRegF dst, vRegF src)
5929 %{
5930 single_instruction;
5931 src : S1(read);
5932 dst : S5(write);
5933 INS01 : ISS;
5934 NEON_FP : S5;
5935 %}
5936
5937 pipe_class fp_uop_d(vRegD dst, vRegD src)
5938 %{
5939 single_instruction;
5940 src : S1(read);
5941 dst : S5(write);
5942 INS01 : ISS;
5943 NEON_FP : S5;
5944 %}
5945
5946 pipe_class fp_d2f(vRegF dst, vRegD src)
5947 %{
5948 single_instruction;
5949 src : S1(read);
5950 dst : S5(write);
5951 INS01 : ISS;
5952 NEON_FP : S5;
5953 %}
5954
5955 pipe_class fp_f2d(vRegD dst, vRegF src)
5956 %{
5957 single_instruction;
5958 src : S1(read);
5959 dst : S5(write);
5960 INS01 : ISS;
5961 NEON_FP : S5;
5962 %}
5963
5964 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5965 %{
5966 single_instruction;
5967 src : S1(read);
5968 dst : S5(write);
5969 INS01 : ISS;
5970 NEON_FP : S5;
5971 %}
5972
5973 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5974 %{
5975 single_instruction;
5976 src : S1(read);
5977 dst : S5(write);
5978 INS01 : ISS;
5979 NEON_FP : S5;
5980 %}
5981
5982 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5983 %{
5984 single_instruction;
5985 src : S1(read);
5986 dst : S5(write);
5987 INS01 : ISS;
5988 NEON_FP : S5;
5989 %}
5990
5991 pipe_class fp_l2f(vRegF dst, iRegL src)
5992 %{
5993 single_instruction;
5994 src : S1(read);
5995 dst : S5(write);
5996 INS01 : ISS;
5997 NEON_FP : S5;
5998 %}
5999
6000 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6001 %{
6002 single_instruction;
6003 src : S1(read);
6004 dst : S5(write);
6005 INS01 : ISS;
6006 NEON_FP : S5;
6007 %}
6008
6009 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6010 %{
6011 single_instruction;
6012 src : S1(read);
6013 dst : S5(write);
6014 INS01 : ISS;
6015 NEON_FP : S5;
6016 %}
6017
6018 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6019 %{
6020 single_instruction;
6021 src : S1(read);
6022 dst : S5(write);
6023 INS01 : ISS;
6024 NEON_FP : S5;
6025 %}
6026
6027 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6028 %{
6029 single_instruction;
6030 src : S1(read);
6031 dst : S5(write);
6032 INS01 : ISS;
6033 NEON_FP : S5;
6034 %}
6035
6036 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6037 %{
6038 single_instruction;
6039 src1 : S1(read);
6040 src2 : S2(read);
6041 dst : S5(write);
6042 INS0 : ISS;
6043 NEON_FP : S5;
6044 %}
6045
6046 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6047 %{
6048 single_instruction;
6049 src1 : S1(read);
6050 src2 : S2(read);
6051 dst : S5(write);
6052 INS0 : ISS;
6053 NEON_FP : S5;
6054 %}
6055
6056 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6057 %{
6058 single_instruction;
6059 cr : S1(read);
6060 src1 : S1(read);
6061 src2 : S1(read);
6062 dst : S3(write);
6063 INS01 : ISS;
6064 NEON_FP : S3;
6065 %}
6066
6067 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6068 %{
6069 single_instruction;
6070 cr : S1(read);
6071 src1 : S1(read);
6072 src2 : S1(read);
6073 dst : S3(write);
6074 INS01 : ISS;
6075 NEON_FP : S3;
6076 %}
6077
6078 pipe_class fp_imm_s(vRegF dst)
6079 %{
6080 single_instruction;
6081 dst : S3(write);
6082 INS01 : ISS;
6083 NEON_FP : S3;
6084 %}
6085
6086 pipe_class fp_imm_d(vRegD dst)
6087 %{
6088 single_instruction;
6089 dst : S3(write);
6090 INS01 : ISS;
6091 NEON_FP : S3;
6092 %}
6093
6094 pipe_class fp_load_constant_s(vRegF dst)
6095 %{
6096 single_instruction;
6097 dst : S4(write);
6098 INS01 : ISS;
6099 NEON_FP : S4;
6100 %}
6101
6102 pipe_class fp_load_constant_d(vRegD dst)
6103 %{
6104 single_instruction;
6105 dst : S4(write);
6106 INS01 : ISS;
6107 NEON_FP : S4;
6108 %}
6109
6110 //------- Integer ALU operations --------------------------
6111
6112 // Integer ALU reg-reg operation
6113 // Operands needed in EX1, result generated in EX2
6114 // Eg. ADD x0, x1, x2
6115 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6116 %{
6117 single_instruction;
6118 dst : EX2(write);
6119 src1 : EX1(read);
6120 src2 : EX1(read);
6121 INS01 : ISS; // Dual issue as instruction 0 or 1
6122 ALU : EX2;
6123 %}
6124
6125 // Integer ALU reg-reg operation with constant shift
6126 // Shifted register must be available in LATE_ISS instead of EX1
6127 // Eg. ADD x0, x1, x2, LSL #2
6128 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6129 %{
6130 single_instruction;
6131 dst : EX2(write);
6132 src1 : EX1(read);
6133 src2 : ISS(read);
6134 INS01 : ISS;
6135 ALU : EX2;
6136 %}
6137
6138 // Integer ALU reg operation with constant shift
6139 // Eg. LSL x0, x1, #shift
6140 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6141 %{
6142 single_instruction;
6143 dst : EX2(write);
6144 src1 : ISS(read);
6145 INS01 : ISS;
6146 ALU : EX2;
6147 %}
6148
6149 // Integer ALU reg-reg operation with variable shift
6150 // Both operands must be available in LATE_ISS instead of EX1
6151 // Result is available in EX1 instead of EX2
6152 // Eg. LSLV x0, x1, x2
6153 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6154 %{
6155 single_instruction;
6156 dst : EX1(write);
6157 src1 : ISS(read);
6158 src2 : ISS(read);
6159 INS01 : ISS;
6160 ALU : EX1;
6161 %}
6162
6163 // Integer ALU reg-reg operation with extract
6164 // As for _vshift above, but result generated in EX2
6165 // Eg. EXTR x0, x1, x2, #N
6166 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6167 %{
6168 single_instruction;
6169 dst : EX2(write);
6170 src1 : ISS(read);
6171 src2 : ISS(read);
6172 INS1 : ISS; // Can only dual issue as Instruction 1
6173 ALU : EX1;
6174 %}
6175
6176 // Integer ALU reg operation
6177 // Eg. NEG x0, x1
6178 pipe_class ialu_reg(iRegI dst, iRegI src)
6179 %{
6180 single_instruction;
6181 dst : EX2(write);
6182 src : EX1(read);
6183 INS01 : ISS;
6184 ALU : EX2;
6185 %}
6186
6187 // Integer ALU reg mmediate operation
6188 // Eg. ADD x0, x1, #N
6189 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6190 %{
6191 single_instruction;
6192 dst : EX2(write);
6193 src1 : EX1(read);
6194 INS01 : ISS;
6195 ALU : EX2;
6196 %}
6197
6198 // Integer ALU immediate operation (no source operands)
6199 // Eg. MOV x0, #N
6200 pipe_class ialu_imm(iRegI dst)
6201 %{
6202 single_instruction;
6203 dst : EX1(write);
6204 INS01 : ISS;
6205 ALU : EX1;
6206 %}
6207
6208 //------- Compare operation -------------------------------
6209
6210 // Compare reg-reg
6211 // Eg. CMP x0, x1
6212 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6213 %{
6214 single_instruction;
6215 // fixed_latency(16);
6216 cr : EX2(write);
6217 op1 : EX1(read);
6218 op2 : EX1(read);
6219 INS01 : ISS;
6220 ALU : EX2;
6221 %}
6222
6223 // Compare reg-reg
6224 // Eg. CMP x0, #N
6225 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6226 %{
6227 single_instruction;
6228 // fixed_latency(16);
6229 cr : EX2(write);
6230 op1 : EX1(read);
6231 INS01 : ISS;
6232 ALU : EX2;
6233 %}
6234
6235 //------- Conditional instructions ------------------------
6236
6237 // Conditional no operands
6238 // Eg. CSINC x0, zr, zr, <cond>
6239 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6240 %{
6241 single_instruction;
6242 cr : EX1(read);
6243 dst : EX2(write);
6244 INS01 : ISS;
6245 ALU : EX2;
6246 %}
6247
6248 // Conditional 2 operand
6249 // EG. CSEL X0, X1, X2, <cond>
6250 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6251 %{
6252 single_instruction;
6253 cr : EX1(read);
6254 src1 : EX1(read);
6255 src2 : EX1(read);
6256 dst : EX2(write);
6257 INS01 : ISS;
6258 ALU : EX2;
6259 %}
6260
6261 // Conditional 2 operand
6262 // EG. CSEL X0, X1, X2, <cond>
6263 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6264 %{
6265 single_instruction;
6266 cr : EX1(read);
6267 src : EX1(read);
6268 dst : EX2(write);
6269 INS01 : ISS;
6270 ALU : EX2;
6271 %}
6272
6273 //------- Multiply pipeline operations --------------------
6274
6275 // Multiply reg-reg
6276 // Eg. MUL w0, w1, w2
6277 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6278 %{
6279 single_instruction;
6280 dst : WR(write);
6281 src1 : ISS(read);
6282 src2 : ISS(read);
6283 INS01 : ISS;
6284 MAC : WR;
6285 %}
6286
6287 // Multiply accumulate
6288 // Eg. MADD w0, w1, w2, w3
6289 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6290 %{
6291 single_instruction;
6292 dst : WR(write);
6293 src1 : ISS(read);
6294 src2 : ISS(read);
6295 src3 : ISS(read);
6296 INS01 : ISS;
6297 MAC : WR;
6298 %}
6299
6300 // Eg. MUL w0, w1, w2
6301 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6302 %{
6303 single_instruction;
6304 fixed_latency(3); // Maximum latency for 64 bit mul
6305 dst : WR(write);
6306 src1 : ISS(read);
6307 src2 : ISS(read);
6308 INS01 : ISS;
6309 MAC : WR;
6310 %}
6311
6312 // Multiply accumulate
6313 // Eg. MADD w0, w1, w2, w3
6314 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6315 %{
6316 single_instruction;
6317 fixed_latency(3); // Maximum latency for 64 bit mul
6318 dst : WR(write);
6319 src1 : ISS(read);
6320 src2 : ISS(read);
6321 src3 : ISS(read);
6322 INS01 : ISS;
6323 MAC : WR;
6324 %}
6325
6326 //------- Divide pipeline operations --------------------
6327
6328 // Eg. SDIV w0, w1, w2
6329 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6330 %{
6331 single_instruction;
6332 fixed_latency(8); // Maximum latency for 32 bit divide
6333 dst : WR(write);
6334 src1 : ISS(read);
6335 src2 : ISS(read);
6336 INS0 : ISS; // Can only dual issue as instruction 0
6337 DIV : WR;
6338 %}
6339
6340 // Eg. SDIV x0, x1, x2
6341 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6342 %{
6343 single_instruction;
6344 fixed_latency(16); // Maximum latency for 64 bit divide
6345 dst : WR(write);
6346 src1 : ISS(read);
6347 src2 : ISS(read);
6348 INS0 : ISS; // Can only dual issue as instruction 0
6349 DIV : WR;
6350 %}
6351
6352 //------- Load pipeline operations ------------------------
6353
6354 // Load - prefetch
6355 // Eg. PFRM <mem>
6356 pipe_class iload_prefetch(memory mem)
6357 %{
6358 single_instruction;
6359 mem : ISS(read);
6360 INS01 : ISS;
6361 LDST : WR;
6362 %}
6363
6364 // Load - reg, mem
6365 // Eg. LDR x0, <mem>
6366 pipe_class iload_reg_mem(iRegI dst, memory mem)
6367 %{
6368 single_instruction;
6369 dst : WR(write);
6370 mem : ISS(read);
6371 INS01 : ISS;
6372 LDST : WR;
6373 %}
6374
6375 // Load - reg, reg
6376 // Eg. LDR x0, [sp, x1]
6377 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6378 %{
6379 single_instruction;
6380 dst : WR(write);
6381 src : ISS(read);
6382 INS01 : ISS;
6383 LDST : WR;
6384 %}
6385
6386 //------- Store pipeline operations -----------------------
6387
6388 // Store - zr, mem
6389 // Eg. STR zr, <mem>
6390 pipe_class istore_mem(memory mem)
6391 %{
6392 single_instruction;
6393 mem : ISS(read);
6394 INS01 : ISS;
6395 LDST : WR;
6396 %}
6397
6398 // Store - reg, mem
6399 // Eg. STR x0, <mem>
6400 pipe_class istore_reg_mem(iRegI src, memory mem)
6401 %{
6402 single_instruction;
6403 mem : ISS(read);
6404 src : EX2(read);
6405 INS01 : ISS;
6406 LDST : WR;
6407 %}
6408
6409 // Store - reg, reg
6410 // Eg. STR x0, [sp, x1]
6411 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6412 %{
6413 single_instruction;
6414 dst : ISS(read);
6415 src : EX2(read);
6416 INS01 : ISS;
6417 LDST : WR;
6418 %}
6419
6420 //------- Store pipeline operations -----------------------
6421
6422 // Branch
6423 pipe_class pipe_branch()
6424 %{
6425 single_instruction;
6426 INS01 : ISS;
6427 BRANCH : EX1;
6428 %}
6429
6430 // Conditional branch
6431 pipe_class pipe_branch_cond(rFlagsReg cr)
6432 %{
6433 single_instruction;
6434 cr : EX1(read);
6435 INS01 : ISS;
6436 BRANCH : EX1;
6437 %}
6438
6439 // Compare & Branch
6440 // EG. CBZ/CBNZ
6441 pipe_class pipe_cmp_branch(iRegI op1)
6442 %{
6443 single_instruction;
6444 op1 : EX1(read);
6445 INS01 : ISS;
6446 BRANCH : EX1;
6447 %}
6448
6449 //------- Synchronisation operations ----------------------
6450
6451 // Any operation requiring serialization.
6452 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6453 pipe_class pipe_serial()
6454 %{
6455 single_instruction;
6456 force_serialization;
6457 fixed_latency(16);
6458 INS01 : ISS(2); // Cannot dual issue with any other instruction
6459 LDST : WR;
6460 %}
6461
6462 // Generic big/slow expanded idiom - also serialized
6463 pipe_class pipe_slow()
6464 %{
6465 instruction_count(10);
6466 multiple_bundles;
6467 force_serialization;
6468 fixed_latency(16);
6469 INS01 : ISS(2); // Cannot dual issue with any other instruction
6470 LDST : WR;
6471 %}
6472
6473 // Empty pipeline class
6474 pipe_class pipe_class_empty()
6475 %{
6476 single_instruction;
6477 fixed_latency(0);
6478 %}
6479
6480 // Default pipeline class.
6481 pipe_class pipe_class_default()
6482 %{
6483 single_instruction;
6484 fixed_latency(2);
6485 %}
6486
6487 // Pipeline class for compares.
6488 pipe_class pipe_class_compare()
6489 %{
6490 single_instruction;
6491 fixed_latency(16);
6492 %}
6493
6494 // Pipeline class for memory operations.
6495 pipe_class pipe_class_memory()
6496 %{
6497 single_instruction;
6498 fixed_latency(16);
6499 %}
6500
6501 // Pipeline class for call.
6502 pipe_class pipe_class_call()
6503 %{
6504 single_instruction;
6505 fixed_latency(100);
6506 %}
6507
6508 // Define the class for the Nop node.
6509 define %{
6510 MachNop = pipe_class_empty;
6511 %}
6512
6513 %}
6514 //----------INSTRUCTIONS-------------------------------------------------------
6515 //
6516 // match -- States which machine-independent subtree may be replaced
6517 // by this instruction.
6518 // ins_cost -- The estimated cost of this instruction is used by instruction
6519 // selection to identify a minimum cost tree of machine
6520 // instructions that matches a tree of machine-independent
6521 // instructions.
6522 // format -- A string providing the disassembly for this instruction.
6523 // The value of an instruction's operand may be inserted
6524 // by referring to it with a '$' prefix.
6525 // opcode -- Three instruction opcodes may be provided. These are referred
6526 // to within an encode class as $primary, $secondary, and $tertiary
6527 // rrspectively. The primary opcode is commonly used to
6528 // indicate the type of machine instruction, while secondary
6529 // and tertiary are often used for prefix options or addressing
6530 // modes.
6531 // ins_encode -- A list of encode classes with parameters. The encode class
6532 // name must have been defined in an 'enc_class' specification
6533 // in the encode section of the architecture description.
6534
6535 // ============================================================================
6536 // Memory (Load/Store) Instructions
6537
6538 // Load Instructions
6539
6540 // Load Byte (8 bit signed)
6541 instruct loadB(iRegINoSp dst, memory1 mem)
6542 %{
6543 match(Set dst (LoadB mem));
6544 predicate(!needs_acquiring_load(n));
6545
6546 ins_cost(4 * INSN_COST);
6547 format %{ "ldrsbw $dst, $mem\t# byte" %}
6548
6549 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6550
6551 ins_pipe(iload_reg_mem);
6552 %}
6553
6554 // Load Byte (8 bit signed) into long
6555 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6556 %{
6557 match(Set dst (ConvI2L (LoadB mem)));
6558 predicate(!needs_acquiring_load(n->in(1)));
6559
6560 ins_cost(4 * INSN_COST);
6561 format %{ "ldrsb $dst, $mem\t# byte" %}
6562
6563 ins_encode(aarch64_enc_ldrsb(dst, mem));
6564
6565 ins_pipe(iload_reg_mem);
6566 %}
6567
6568 // Load Byte (8 bit unsigned)
6569 instruct loadUB(iRegINoSp dst, memory1 mem)
6570 %{
6571 match(Set dst (LoadUB mem));
6572 predicate(!needs_acquiring_load(n));
6573
6574 ins_cost(4 * INSN_COST);
6575 format %{ "ldrbw $dst, $mem\t# byte" %}
6576
6577 ins_encode(aarch64_enc_ldrb(dst, mem));
6578
6579 ins_pipe(iload_reg_mem);
6580 %}
6581
6582 // Load Byte (8 bit unsigned) into long
6583 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6584 %{
6585 match(Set dst (ConvI2L (LoadUB mem)));
6586 predicate(!needs_acquiring_load(n->in(1)));
6587
6588 ins_cost(4 * INSN_COST);
6589 format %{ "ldrb $dst, $mem\t# byte" %}
6590
6591 ins_encode(aarch64_enc_ldrb(dst, mem));
6592
6593 ins_pipe(iload_reg_mem);
6594 %}
6595
6596 // Load Short (16 bit signed)
6597 instruct loadS(iRegINoSp dst, memory2 mem)
6598 %{
6599 match(Set dst (LoadS mem));
6600 predicate(!needs_acquiring_load(n));
6601
6602 ins_cost(4 * INSN_COST);
6603 format %{ "ldrshw $dst, $mem\t# short" %}
6604
6605 ins_encode(aarch64_enc_ldrshw(dst, mem));
6606
6607 ins_pipe(iload_reg_mem);
6608 %}
6609
6610 // Load Short (16 bit signed) into long
6611 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6612 %{
6613 match(Set dst (ConvI2L (LoadS mem)));
6614 predicate(!needs_acquiring_load(n->in(1)));
6615
6616 ins_cost(4 * INSN_COST);
6617 format %{ "ldrsh $dst, $mem\t# short" %}
6618
6619 ins_encode(aarch64_enc_ldrsh(dst, mem));
6620
6621 ins_pipe(iload_reg_mem);
6622 %}
6623
6624 // Load Char (16 bit unsigned)
6625 instruct loadUS(iRegINoSp dst, memory2 mem)
6626 %{
6627 match(Set dst (LoadUS mem));
6628 predicate(!needs_acquiring_load(n));
6629
6630 ins_cost(4 * INSN_COST);
6631 format %{ "ldrh $dst, $mem\t# short" %}
6632
6633 ins_encode(aarch64_enc_ldrh(dst, mem));
6634
6635 ins_pipe(iload_reg_mem);
6636 %}
6637
6638 // Load Short/Char (16 bit unsigned) into long
6639 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6640 %{
6641 match(Set dst (ConvI2L (LoadUS mem)));
6642 predicate(!needs_acquiring_load(n->in(1)));
6643
6644 ins_cost(4 * INSN_COST);
6645 format %{ "ldrh $dst, $mem\t# short" %}
6646
6647 ins_encode(aarch64_enc_ldrh(dst, mem));
6648
6649 ins_pipe(iload_reg_mem);
6650 %}
6651
6652 // Load Integer (32 bit signed)
6653 instruct loadI(iRegINoSp dst, memory4 mem)
6654 %{
6655 match(Set dst (LoadI mem));
6656 predicate(!needs_acquiring_load(n));
6657
6658 ins_cost(4 * INSN_COST);
6659 format %{ "ldrw $dst, $mem\t# int" %}
6660
6661 ins_encode(aarch64_enc_ldrw(dst, mem));
6662
6663 ins_pipe(iload_reg_mem);
6664 %}
6665
6666 // Load Integer (32 bit signed) into long
6667 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6668 %{
6669 match(Set dst (ConvI2L (LoadI mem)));
6670 predicate(!needs_acquiring_load(n->in(1)));
6671
6672 ins_cost(4 * INSN_COST);
6673 format %{ "ldrsw $dst, $mem\t# int" %}
6674
6675 ins_encode(aarch64_enc_ldrsw(dst, mem));
6676
6677 ins_pipe(iload_reg_mem);
6678 %}
6679
6680 // Load Integer (32 bit unsigned) into long
6681 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6682 %{
6683 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6684 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6685
6686 ins_cost(4 * INSN_COST);
6687 format %{ "ldrw $dst, $mem\t# int" %}
6688
6689 ins_encode(aarch64_enc_ldrw(dst, mem));
6690
6691 ins_pipe(iload_reg_mem);
6692 %}
6693
6694 // Load Long (64 bit signed)
6695 instruct loadL(iRegLNoSp dst, memory8 mem)
6696 %{
6697 match(Set dst (LoadL mem));
6698 predicate(!needs_acquiring_load(n));
6699
6700 ins_cost(4 * INSN_COST);
6701 format %{ "ldr $dst, $mem\t# int" %}
6702
6703 ins_encode(aarch64_enc_ldr(dst, mem));
6704
6705 ins_pipe(iload_reg_mem);
6706 %}
6707
6708 // Load Range
6709 instruct loadRange(iRegINoSp dst, memory4 mem)
6710 %{
6711 match(Set dst (LoadRange mem));
6712
6713 ins_cost(4 * INSN_COST);
6714 format %{ "ldrw $dst, $mem\t# range" %}
6715
6716 ins_encode(aarch64_enc_ldrw(dst, mem));
6717
6718 ins_pipe(iload_reg_mem);
6719 %}
6720
6721 // Load Pointer
6722 instruct loadP(iRegPNoSp dst, memory8 mem)
6723 %{
6724 match(Set dst (LoadP mem));
6725 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6726
6727 ins_cost(4 * INSN_COST);
6728 format %{ "ldr $dst, $mem\t# ptr" %}
6729
6730 ins_encode(aarch64_enc_ldr(dst, mem));
6731
6732 ins_pipe(iload_reg_mem);
6733 %}
6734
6735 // Load Compressed Pointer
6736 instruct loadN(iRegNNoSp dst, memory4 mem)
6737 %{
6738 match(Set dst (LoadN mem));
6739 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6740
6741 ins_cost(4 * INSN_COST);
6742 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6743
6744 ins_encode(aarch64_enc_ldrw(dst, mem));
6745
6746 ins_pipe(iload_reg_mem);
6747 %}
6748
6749 // Load Klass Pointer
6750 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6751 %{
6752 match(Set dst (LoadKlass mem));
6753 predicate(!needs_acquiring_load(n));
6754
6755 ins_cost(4 * INSN_COST);
6756 format %{ "ldr $dst, $mem\t# class" %}
6757
6758 ins_encode(aarch64_enc_ldr(dst, mem));
6759
6760 ins_pipe(iload_reg_mem);
6761 %}
6762
6763 // Load Narrow Klass Pointer
6764 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6765 %{
6766 match(Set dst (LoadNKlass mem));
6767 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6768
6769 ins_cost(4 * INSN_COST);
6770 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6771
6772 ins_encode(aarch64_enc_ldrw(dst, mem));
6773
6774 ins_pipe(iload_reg_mem);
6775 %}
6776
6777 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6778 %{
6779 match(Set dst (LoadNKlass mem));
6780 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6781
6782 ins_cost(4 * INSN_COST);
6783 format %{
6784 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6785 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6786 %}
6787 ins_encode %{
6788 // inlined aarch64_enc_ldrw
6789 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6790 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6791 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6792 %}
6793 ins_pipe(iload_reg_mem);
6794 %}
6795
6796 // Load Float
6797 instruct loadF(vRegF dst, memory4 mem)
6798 %{
6799 match(Set dst (LoadF mem));
6800 predicate(!needs_acquiring_load(n));
6801
6802 ins_cost(4 * INSN_COST);
6803 format %{ "ldrs $dst, $mem\t# float" %}
6804
6805 ins_encode( aarch64_enc_ldrs(dst, mem) );
6806
6807 ins_pipe(pipe_class_memory);
6808 %}
6809
6810 // Load Double
6811 instruct loadD(vRegD dst, memory8 mem)
6812 %{
6813 match(Set dst (LoadD mem));
6814 predicate(!needs_acquiring_load(n));
6815
6816 ins_cost(4 * INSN_COST);
6817 format %{ "ldrd $dst, $mem\t# double" %}
6818
6819 ins_encode( aarch64_enc_ldrd(dst, mem) );
6820
6821 ins_pipe(pipe_class_memory);
6822 %}
6823
6824
6825 // Load Int Constant
6826 instruct loadConI(iRegINoSp dst, immI src)
6827 %{
6828 match(Set dst src);
6829
6830 ins_cost(INSN_COST);
6831 format %{ "mov $dst, $src\t# int" %}
6832
6833 ins_encode( aarch64_enc_movw_imm(dst, src) );
6834
6835 ins_pipe(ialu_imm);
6836 %}
6837
6838 // Load Long Constant
6839 instruct loadConL(iRegLNoSp dst, immL src)
6840 %{
6841 match(Set dst src);
6842
6843 ins_cost(INSN_COST);
6844 format %{ "mov $dst, $src\t# long" %}
6845
6846 ins_encode( aarch64_enc_mov_imm(dst, src) );
6847
6848 ins_pipe(ialu_imm);
6849 %}
6850
6851 // Load Pointer Constant
6852
6853 instruct loadConP(iRegPNoSp dst, immP con)
6854 %{
6855 match(Set dst con);
6856
6857 ins_cost(INSN_COST * 4);
6858 format %{
6859 "mov $dst, $con\t# ptr\n\t"
6860 %}
6861
6862 ins_encode(aarch64_enc_mov_p(dst, con));
6863
6864 ins_pipe(ialu_imm);
6865 %}
6866
6867 // Load Null Pointer Constant
6868
6869 instruct loadConP0(iRegPNoSp dst, immP0 con)
6870 %{
6871 match(Set dst con);
6872
6873 ins_cost(INSN_COST);
6874 format %{ "mov $dst, $con\t# nullptr ptr" %}
6875
6876 ins_encode(aarch64_enc_mov_p0(dst, con));
6877
6878 ins_pipe(ialu_imm);
6879 %}
6880
6881 // Load Pointer Constant One
6882
6883 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6884 %{
6885 match(Set dst con);
6886
6887 ins_cost(INSN_COST);
6888 format %{ "mov $dst, $con\t# nullptr ptr" %}
6889
6890 ins_encode(aarch64_enc_mov_p1(dst, con));
6891
6892 ins_pipe(ialu_imm);
6893 %}
6894
6895 // Load Narrow Pointer Constant
6896
6897 instruct loadConN(iRegNNoSp dst, immN con)
6898 %{
6899 match(Set dst con);
6900
6901 ins_cost(INSN_COST * 4);
6902 format %{ "mov $dst, $con\t# compressed ptr" %}
6903
6904 ins_encode(aarch64_enc_mov_n(dst, con));
6905
6906 ins_pipe(ialu_imm);
6907 %}
6908
6909 // Load Narrow Null Pointer Constant
6910
6911 instruct loadConN0(iRegNNoSp dst, immN0 con)
6912 %{
6913 match(Set dst con);
6914
6915 ins_cost(INSN_COST);
6916 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6917
6918 ins_encode(aarch64_enc_mov_n0(dst, con));
6919
6920 ins_pipe(ialu_imm);
6921 %}
6922
6923 // Load Narrow Klass Constant
6924
6925 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6926 %{
6927 match(Set dst con);
6928
6929 ins_cost(INSN_COST);
6930 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6931
6932 ins_encode(aarch64_enc_mov_nk(dst, con));
6933
6934 ins_pipe(ialu_imm);
6935 %}
6936
6937 // Load Packed Float Constant
6938
6939 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6940 match(Set dst con);
6941 ins_cost(INSN_COST * 4);
6942 format %{ "fmovs $dst, $con"%}
6943 ins_encode %{
6944 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6945 %}
6946
6947 ins_pipe(fp_imm_s);
6948 %}
6949
6950 // Load Float Constant
6951
6952 instruct loadConF(vRegF dst, immF con) %{
6953 match(Set dst con);
6954
6955 ins_cost(INSN_COST * 4);
6956
6957 format %{
6958 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6959 %}
6960
6961 ins_encode %{
6962 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6963 %}
6964
6965 ins_pipe(fp_load_constant_s);
6966 %}
6967
6968 // Load Packed Double Constant
6969
6970 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6971 match(Set dst con);
6972 ins_cost(INSN_COST);
6973 format %{ "fmovd $dst, $con"%}
6974 ins_encode %{
6975 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6976 %}
6977
6978 ins_pipe(fp_imm_d);
6979 %}
6980
6981 // Load Double Constant
6982
6983 instruct loadConD(vRegD dst, immD con) %{
6984 match(Set dst con);
6985
6986 ins_cost(INSN_COST * 5);
6987 format %{
6988 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6989 %}
6990
6991 ins_encode %{
6992 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6993 %}
6994
6995 ins_pipe(fp_load_constant_d);
6996 %}
6997
6998 // Load Half Float Constant
6999 instruct loadConH(vRegF dst, immH con) %{
7000 match(Set dst con);
7001 format %{ "mov rscratch1, $con\n\t"
7002 "fmov $dst, rscratch1"
7003 %}
7004 ins_encode %{
7005 __ movw(rscratch1, (uint32_t)$con$$constant);
7006 __ fmovs($dst$$FloatRegister, rscratch1);
7007 %}
7008 ins_pipe(pipe_class_default);
7009 %}
7010
7011 // Store Instructions
7012
7013 // Store Byte
7014 instruct storeB(iRegIorL2I src, memory1 mem)
7015 %{
7016 match(Set mem (StoreB mem src));
7017 predicate(!needs_releasing_store(n));
7018
7019 ins_cost(INSN_COST);
7020 format %{ "strb $src, $mem\t# byte" %}
7021
7022 ins_encode(aarch64_enc_strb(src, mem));
7023
7024 ins_pipe(istore_reg_mem);
7025 %}
7026
7027
7028 instruct storeimmB0(immI0 zero, memory1 mem)
7029 %{
7030 match(Set mem (StoreB mem zero));
7031 predicate(!needs_releasing_store(n));
7032
7033 ins_cost(INSN_COST);
7034 format %{ "strb rscractch2, $mem\t# byte" %}
7035
7036 ins_encode(aarch64_enc_strb0(mem));
7037
7038 ins_pipe(istore_mem);
7039 %}
7040
7041 // Store Char/Short
7042 instruct storeC(iRegIorL2I src, memory2 mem)
7043 %{
7044 match(Set mem (StoreC mem src));
7045 predicate(!needs_releasing_store(n));
7046
7047 ins_cost(INSN_COST);
7048 format %{ "strh $src, $mem\t# short" %}
7049
7050 ins_encode(aarch64_enc_strh(src, mem));
7051
7052 ins_pipe(istore_reg_mem);
7053 %}
7054
7055 instruct storeimmC0(immI0 zero, memory2 mem)
7056 %{
7057 match(Set mem (StoreC mem zero));
7058 predicate(!needs_releasing_store(n));
7059
7060 ins_cost(INSN_COST);
7061 format %{ "strh zr, $mem\t# short" %}
7062
7063 ins_encode(aarch64_enc_strh0(mem));
7064
7065 ins_pipe(istore_mem);
7066 %}
7067
7068 // Store Integer
7069
7070 instruct storeI(iRegIorL2I src, memory4 mem)
7071 %{
7072 match(Set mem(StoreI mem src));
7073 predicate(!needs_releasing_store(n));
7074
7075 ins_cost(INSN_COST);
7076 format %{ "strw $src, $mem\t# int" %}
7077
7078 ins_encode(aarch64_enc_strw(src, mem));
7079
7080 ins_pipe(istore_reg_mem);
7081 %}
7082
7083 instruct storeimmI0(immI0 zero, memory4 mem)
7084 %{
7085 match(Set mem(StoreI mem zero));
7086 predicate(!needs_releasing_store(n));
7087
7088 ins_cost(INSN_COST);
7089 format %{ "strw zr, $mem\t# int" %}
7090
7091 ins_encode(aarch64_enc_strw0(mem));
7092
7093 ins_pipe(istore_mem);
7094 %}
7095
7096 // Store Long (64 bit signed)
7097 instruct storeL(iRegL src, memory8 mem)
7098 %{
7099 match(Set mem (StoreL mem src));
7100 predicate(!needs_releasing_store(n));
7101
7102 ins_cost(INSN_COST);
7103 format %{ "str $src, $mem\t# int" %}
7104
7105 ins_encode(aarch64_enc_str(src, mem));
7106
7107 ins_pipe(istore_reg_mem);
7108 %}
7109
7110 // Store Long (64 bit signed)
7111 instruct storeimmL0(immL0 zero, memory8 mem)
7112 %{
7113 match(Set mem (StoreL mem zero));
7114 predicate(!needs_releasing_store(n));
7115
7116 ins_cost(INSN_COST);
7117 format %{ "str zr, $mem\t# int" %}
7118
7119 ins_encode(aarch64_enc_str0(mem));
7120
7121 ins_pipe(istore_mem);
7122 %}
7123
7124 // Store Pointer
7125 instruct storeP(iRegP src, memory8 mem)
7126 %{
7127 match(Set mem (StoreP mem src));
7128 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7129
7130 ins_cost(INSN_COST);
7131 format %{ "str $src, $mem\t# ptr" %}
7132
7133 ins_encode(aarch64_enc_str(src, mem));
7134
7135 ins_pipe(istore_reg_mem);
7136 %}
7137
7138 // Store Pointer
7139 instruct storeimmP0(immP0 zero, memory8 mem)
7140 %{
7141 match(Set mem (StoreP mem zero));
7142 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7143
7144 ins_cost(INSN_COST);
7145 format %{ "str zr, $mem\t# ptr" %}
7146
7147 ins_encode(aarch64_enc_str0(mem));
7148
7149 ins_pipe(istore_mem);
7150 %}
7151
7152 // Store Compressed Pointer
7153 instruct storeN(iRegN src, memory4 mem)
7154 %{
7155 match(Set mem (StoreN mem src));
7156 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7157
7158 ins_cost(INSN_COST);
7159 format %{ "strw $src, $mem\t# compressed ptr" %}
7160
7161 ins_encode(aarch64_enc_strw(src, mem));
7162
7163 ins_pipe(istore_reg_mem);
7164 %}
7165
7166 instruct storeImmN0(immN0 zero, memory4 mem)
7167 %{
7168 match(Set mem (StoreN mem zero));
7169 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7170
7171 ins_cost(INSN_COST);
7172 format %{ "strw zr, $mem\t# compressed ptr" %}
7173
7174 ins_encode(aarch64_enc_strw0(mem));
7175
7176 ins_pipe(istore_mem);
7177 %}
7178
7179 // Store Float
7180 instruct storeF(vRegF src, memory4 mem)
7181 %{
7182 match(Set mem (StoreF mem src));
7183 predicate(!needs_releasing_store(n));
7184
7185 ins_cost(INSN_COST);
7186 format %{ "strs $src, $mem\t# float" %}
7187
7188 ins_encode( aarch64_enc_strs(src, mem) );
7189
7190 ins_pipe(pipe_class_memory);
7191 %}
7192
7193 // TODO
7194 // implement storeImmF0 and storeFImmPacked
7195
7196 // Store Double
7197 instruct storeD(vRegD src, memory8 mem)
7198 %{
7199 match(Set mem (StoreD mem src));
7200 predicate(!needs_releasing_store(n));
7201
7202 ins_cost(INSN_COST);
7203 format %{ "strd $src, $mem\t# double" %}
7204
7205 ins_encode( aarch64_enc_strd(src, mem) );
7206
7207 ins_pipe(pipe_class_memory);
7208 %}
7209
7210 // Store Compressed Klass Pointer
7211 instruct storeNKlass(iRegN src, memory4 mem)
7212 %{
7213 predicate(!needs_releasing_store(n));
7214 match(Set mem (StoreNKlass mem src));
7215
7216 ins_cost(INSN_COST);
7217 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7218
7219 ins_encode(aarch64_enc_strw(src, mem));
7220
7221 ins_pipe(istore_reg_mem);
7222 %}
7223
7224 // TODO
7225 // implement storeImmD0 and storeDImmPacked
7226
7227 // prefetch instructions
7228 // Must be safe to execute with invalid address (cannot fault).
7229
7230 instruct prefetchalloc( memory8 mem ) %{
7231 match(PrefetchAllocation mem);
7232
7233 ins_cost(INSN_COST);
7234 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7235
7236 ins_encode( aarch64_enc_prefetchw(mem) );
7237
7238 ins_pipe(iload_prefetch);
7239 %}
7240
7241 // ---------------- volatile loads and stores ----------------
7242
7243 // Load Byte (8 bit signed)
7244 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7245 %{
7246 match(Set dst (LoadB mem));
7247
7248 ins_cost(VOLATILE_REF_COST);
7249 format %{ "ldarsb $dst, $mem\t# byte" %}
7250
7251 ins_encode(aarch64_enc_ldarsb(dst, mem));
7252
7253 ins_pipe(pipe_serial);
7254 %}
7255
7256 // Load Byte (8 bit signed) into long
7257 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7258 %{
7259 match(Set dst (ConvI2L (LoadB mem)));
7260
7261 ins_cost(VOLATILE_REF_COST);
7262 format %{ "ldarsb $dst, $mem\t# byte" %}
7263
7264 ins_encode(aarch64_enc_ldarsb(dst, mem));
7265
7266 ins_pipe(pipe_serial);
7267 %}
7268
7269 // Load Byte (8 bit unsigned)
7270 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7271 %{
7272 match(Set dst (LoadUB mem));
7273
7274 ins_cost(VOLATILE_REF_COST);
7275 format %{ "ldarb $dst, $mem\t# byte" %}
7276
7277 ins_encode(aarch64_enc_ldarb(dst, mem));
7278
7279 ins_pipe(pipe_serial);
7280 %}
7281
7282 // Load Byte (8 bit unsigned) into long
7283 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7284 %{
7285 match(Set dst (ConvI2L (LoadUB mem)));
7286
7287 ins_cost(VOLATILE_REF_COST);
7288 format %{ "ldarb $dst, $mem\t# byte" %}
7289
7290 ins_encode(aarch64_enc_ldarb(dst, mem));
7291
7292 ins_pipe(pipe_serial);
7293 %}
7294
7295 // Load Short (16 bit signed)
7296 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7297 %{
7298 match(Set dst (LoadS mem));
7299
7300 ins_cost(VOLATILE_REF_COST);
7301 format %{ "ldarshw $dst, $mem\t# short" %}
7302
7303 ins_encode(aarch64_enc_ldarshw(dst, mem));
7304
7305 ins_pipe(pipe_serial);
7306 %}
7307
7308 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7309 %{
7310 match(Set dst (LoadUS mem));
7311
7312 ins_cost(VOLATILE_REF_COST);
7313 format %{ "ldarhw $dst, $mem\t# short" %}
7314
7315 ins_encode(aarch64_enc_ldarhw(dst, mem));
7316
7317 ins_pipe(pipe_serial);
7318 %}
7319
7320 // Load Short/Char (16 bit unsigned) into long
7321 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7322 %{
7323 match(Set dst (ConvI2L (LoadUS mem)));
7324
7325 ins_cost(VOLATILE_REF_COST);
7326 format %{ "ldarh $dst, $mem\t# short" %}
7327
7328 ins_encode(aarch64_enc_ldarh(dst, mem));
7329
7330 ins_pipe(pipe_serial);
7331 %}
7332
7333 // Load Short/Char (16 bit signed) into long
7334 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7335 %{
7336 match(Set dst (ConvI2L (LoadS mem)));
7337
7338 ins_cost(VOLATILE_REF_COST);
7339 format %{ "ldarh $dst, $mem\t# short" %}
7340
7341 ins_encode(aarch64_enc_ldarsh(dst, mem));
7342
7343 ins_pipe(pipe_serial);
7344 %}
7345
7346 // Load Integer (32 bit signed)
7347 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7348 %{
7349 match(Set dst (LoadI mem));
7350
7351 ins_cost(VOLATILE_REF_COST);
7352 format %{ "ldarw $dst, $mem\t# int" %}
7353
7354 ins_encode(aarch64_enc_ldarw(dst, mem));
7355
7356 ins_pipe(pipe_serial);
7357 %}
7358
7359 // Load Integer (32 bit unsigned) into long
7360 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7361 %{
7362 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7363
7364 ins_cost(VOLATILE_REF_COST);
7365 format %{ "ldarw $dst, $mem\t# int" %}
7366
7367 ins_encode(aarch64_enc_ldarw(dst, mem));
7368
7369 ins_pipe(pipe_serial);
7370 %}
7371
7372 // Load Long (64 bit signed)
7373 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7374 %{
7375 match(Set dst (LoadL mem));
7376
7377 ins_cost(VOLATILE_REF_COST);
7378 format %{ "ldar $dst, $mem\t# int" %}
7379
7380 ins_encode(aarch64_enc_ldar(dst, mem));
7381
7382 ins_pipe(pipe_serial);
7383 %}
7384
7385 // Load Pointer
7386 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7387 %{
7388 match(Set dst (LoadP mem));
7389 predicate(n->as_Load()->barrier_data() == 0);
7390
7391 ins_cost(VOLATILE_REF_COST);
7392 format %{ "ldar $dst, $mem\t# ptr" %}
7393
7394 ins_encode(aarch64_enc_ldar(dst, mem));
7395
7396 ins_pipe(pipe_serial);
7397 %}
7398
7399 // Load Compressed Pointer
7400 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7401 %{
7402 match(Set dst (LoadN mem));
7403 predicate(n->as_Load()->barrier_data() == 0);
7404
7405 ins_cost(VOLATILE_REF_COST);
7406 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7407
7408 ins_encode(aarch64_enc_ldarw(dst, mem));
7409
7410 ins_pipe(pipe_serial);
7411 %}
7412
7413 // Load Float
7414 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7415 %{
7416 match(Set dst (LoadF mem));
7417
7418 ins_cost(VOLATILE_REF_COST);
7419 format %{ "ldars $dst, $mem\t# float" %}
7420
7421 ins_encode( aarch64_enc_fldars(dst, mem) );
7422
7423 ins_pipe(pipe_serial);
7424 %}
7425
7426 // Load Double
7427 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7428 %{
7429 match(Set dst (LoadD mem));
7430
7431 ins_cost(VOLATILE_REF_COST);
7432 format %{ "ldard $dst, $mem\t# double" %}
7433
7434 ins_encode( aarch64_enc_fldard(dst, mem) );
7435
7436 ins_pipe(pipe_serial);
7437 %}
7438
7439 // Store Byte
7440 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7441 %{
7442 match(Set mem (StoreB mem src));
7443
7444 ins_cost(VOLATILE_REF_COST);
7445 format %{ "stlrb $src, $mem\t# byte" %}
7446
7447 ins_encode(aarch64_enc_stlrb(src, mem));
7448
7449 ins_pipe(pipe_class_memory);
7450 %}
7451
7452 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7453 %{
7454 match(Set mem (StoreB mem zero));
7455
7456 ins_cost(VOLATILE_REF_COST);
7457 format %{ "stlrb zr, $mem\t# byte" %}
7458
7459 ins_encode(aarch64_enc_stlrb0(mem));
7460
7461 ins_pipe(pipe_class_memory);
7462 %}
7463
7464 // Store Char/Short
7465 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7466 %{
7467 match(Set mem (StoreC mem src));
7468
7469 ins_cost(VOLATILE_REF_COST);
7470 format %{ "stlrh $src, $mem\t# short" %}
7471
7472 ins_encode(aarch64_enc_stlrh(src, mem));
7473
7474 ins_pipe(pipe_class_memory);
7475 %}
7476
7477 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7478 %{
7479 match(Set mem (StoreC mem zero));
7480
7481 ins_cost(VOLATILE_REF_COST);
7482 format %{ "stlrh zr, $mem\t# short" %}
7483
7484 ins_encode(aarch64_enc_stlrh0(mem));
7485
7486 ins_pipe(pipe_class_memory);
7487 %}
7488
7489 // Store Integer
7490
7491 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7492 %{
7493 match(Set mem(StoreI mem src));
7494
7495 ins_cost(VOLATILE_REF_COST);
7496 format %{ "stlrw $src, $mem\t# int" %}
7497
7498 ins_encode(aarch64_enc_stlrw(src, mem));
7499
7500 ins_pipe(pipe_class_memory);
7501 %}
7502
7503 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7504 %{
7505 match(Set mem(StoreI mem zero));
7506
7507 ins_cost(VOLATILE_REF_COST);
7508 format %{ "stlrw zr, $mem\t# int" %}
7509
7510 ins_encode(aarch64_enc_stlrw0(mem));
7511
7512 ins_pipe(pipe_class_memory);
7513 %}
7514
7515 // Store Long (64 bit signed)
7516 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7517 %{
7518 match(Set mem (StoreL mem src));
7519
7520 ins_cost(VOLATILE_REF_COST);
7521 format %{ "stlr $src, $mem\t# int" %}
7522
7523 ins_encode(aarch64_enc_stlr(src, mem));
7524
7525 ins_pipe(pipe_class_memory);
7526 %}
7527
7528 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7529 %{
7530 match(Set mem (StoreL mem zero));
7531
7532 ins_cost(VOLATILE_REF_COST);
7533 format %{ "stlr zr, $mem\t# int" %}
7534
7535 ins_encode(aarch64_enc_stlr0(mem));
7536
7537 ins_pipe(pipe_class_memory);
7538 %}
7539
7540 // Store Pointer
7541 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7542 %{
7543 match(Set mem (StoreP mem src));
7544 predicate(n->as_Store()->barrier_data() == 0);
7545
7546 ins_cost(VOLATILE_REF_COST);
7547 format %{ "stlr $src, $mem\t# ptr" %}
7548
7549 ins_encode(aarch64_enc_stlr(src, mem));
7550
7551 ins_pipe(pipe_class_memory);
7552 %}
7553
7554 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7555 %{
7556 match(Set mem (StoreP mem zero));
7557 predicate(n->as_Store()->barrier_data() == 0);
7558
7559 ins_cost(VOLATILE_REF_COST);
7560 format %{ "stlr zr, $mem\t# ptr" %}
7561
7562 ins_encode(aarch64_enc_stlr0(mem));
7563
7564 ins_pipe(pipe_class_memory);
7565 %}
7566
7567 // Store Compressed Pointer
7568 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7569 %{
7570 match(Set mem (StoreN mem src));
7571 predicate(n->as_Store()->barrier_data() == 0);
7572
7573 ins_cost(VOLATILE_REF_COST);
7574 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7575
7576 ins_encode(aarch64_enc_stlrw(src, mem));
7577
7578 ins_pipe(pipe_class_memory);
7579 %}
7580
7581 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7582 %{
7583 match(Set mem (StoreN mem zero));
7584 predicate(n->as_Store()->barrier_data() == 0);
7585
7586 ins_cost(VOLATILE_REF_COST);
7587 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7588
7589 ins_encode(aarch64_enc_stlrw0(mem));
7590
7591 ins_pipe(pipe_class_memory);
7592 %}
7593
7594 // Store Float
7595 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7596 %{
7597 match(Set mem (StoreF mem src));
7598
7599 ins_cost(VOLATILE_REF_COST);
7600 format %{ "stlrs $src, $mem\t# float" %}
7601
7602 ins_encode( aarch64_enc_fstlrs(src, mem) );
7603
7604 ins_pipe(pipe_class_memory);
7605 %}
7606
7607 // TODO
7608 // implement storeImmF0 and storeFImmPacked
7609
7610 // Store Double
7611 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7612 %{
7613 match(Set mem (StoreD mem src));
7614
7615 ins_cost(VOLATILE_REF_COST);
7616 format %{ "stlrd $src, $mem\t# double" %}
7617
7618 ins_encode( aarch64_enc_fstlrd(src, mem) );
7619
7620 ins_pipe(pipe_class_memory);
7621 %}
7622
7623 // ---------------- end of volatile loads and stores ----------------
7624
7625 instruct cacheWB(indirect addr)
7626 %{
7627 predicate(VM_Version::supports_data_cache_line_flush());
7628 match(CacheWB addr);
7629
7630 ins_cost(100);
7631 format %{"cache wb $addr" %}
7632 ins_encode %{
7633 assert($addr->index_position() < 0, "should be");
7634 assert($addr$$disp == 0, "should be");
7635 __ cache_wb(Address($addr$$base$$Register, 0));
7636 %}
7637 ins_pipe(pipe_slow); // XXX
7638 %}
7639
7640 instruct cacheWBPreSync()
7641 %{
7642 predicate(VM_Version::supports_data_cache_line_flush());
7643 match(CacheWBPreSync);
7644
7645 ins_cost(100);
7646 format %{"cache wb presync" %}
7647 ins_encode %{
7648 __ cache_wbsync(true);
7649 %}
7650 ins_pipe(pipe_slow); // XXX
7651 %}
7652
7653 instruct cacheWBPostSync()
7654 %{
7655 predicate(VM_Version::supports_data_cache_line_flush());
7656 match(CacheWBPostSync);
7657
7658 ins_cost(100);
7659 format %{"cache wb postsync" %}
7660 ins_encode %{
7661 __ cache_wbsync(false);
7662 %}
7663 ins_pipe(pipe_slow); // XXX
7664 %}
7665
7666 // ============================================================================
7667 // BSWAP Instructions
7668
7669 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7670 match(Set dst (ReverseBytesI src));
7671
7672 ins_cost(INSN_COST);
7673 format %{ "revw $dst, $src" %}
7674
7675 ins_encode %{
7676 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7677 %}
7678
7679 ins_pipe(ialu_reg);
7680 %}
7681
7682 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7683 match(Set dst (ReverseBytesL src));
7684
7685 ins_cost(INSN_COST);
7686 format %{ "rev $dst, $src" %}
7687
7688 ins_encode %{
7689 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7690 %}
7691
7692 ins_pipe(ialu_reg);
7693 %}
7694
7695 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7696 match(Set dst (ReverseBytesUS src));
7697
7698 ins_cost(INSN_COST);
7699 format %{ "rev16w $dst, $src" %}
7700
7701 ins_encode %{
7702 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7703 %}
7704
7705 ins_pipe(ialu_reg);
7706 %}
7707
7708 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7709 match(Set dst (ReverseBytesS src));
7710
7711 ins_cost(INSN_COST);
7712 format %{ "rev16w $dst, $src\n\t"
7713 "sbfmw $dst, $dst, #0, #15" %}
7714
7715 ins_encode %{
7716 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7717 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7718 %}
7719
7720 ins_pipe(ialu_reg);
7721 %}
7722
7723 // ============================================================================
7724 // Zero Count Instructions
7725
7726 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7727 match(Set dst (CountLeadingZerosI src));
7728
7729 ins_cost(INSN_COST);
7730 format %{ "clzw $dst, $src" %}
7731 ins_encode %{
7732 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7733 %}
7734
7735 ins_pipe(ialu_reg);
7736 %}
7737
7738 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7739 match(Set dst (CountLeadingZerosL src));
7740
7741 ins_cost(INSN_COST);
7742 format %{ "clz $dst, $src" %}
7743 ins_encode %{
7744 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7745 %}
7746
7747 ins_pipe(ialu_reg);
7748 %}
7749
7750 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7751 match(Set dst (CountTrailingZerosI src));
7752
7753 ins_cost(INSN_COST * 2);
7754 format %{ "rbitw $dst, $src\n\t"
7755 "clzw $dst, $dst" %}
7756 ins_encode %{
7757 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7758 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7759 %}
7760
7761 ins_pipe(ialu_reg);
7762 %}
7763
7764 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7765 match(Set dst (CountTrailingZerosL src));
7766
7767 ins_cost(INSN_COST * 2);
7768 format %{ "rbit $dst, $src\n\t"
7769 "clz $dst, $dst" %}
7770 ins_encode %{
7771 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7772 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7773 %}
7774
7775 ins_pipe(ialu_reg);
7776 %}
7777
7778 //---------- Population Count Instructions -------------------------------------
7779 //
7780
7781 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7782 match(Set dst (PopCountI src));
7783 effect(TEMP tmp);
7784 ins_cost(INSN_COST * 13);
7785
7786 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7787 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7788 "addv $tmp, $tmp\t# vector (8B)\n\t"
7789 "mov $dst, $tmp\t# vector (1D)" %}
7790 ins_encode %{
7791 __ fmovs($tmp$$FloatRegister, $src$$Register);
7792 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7793 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7794 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7795 %}
7796
7797 ins_pipe(pipe_class_default);
7798 %}
7799
7800 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7801 match(Set dst (PopCountI (LoadI mem)));
7802 effect(TEMP tmp);
7803 ins_cost(INSN_COST * 13);
7804
7805 format %{ "ldrs $tmp, $mem\n\t"
7806 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7807 "addv $tmp, $tmp\t# vector (8B)\n\t"
7808 "mov $dst, $tmp\t# vector (1D)" %}
7809 ins_encode %{
7810 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7811 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7812 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7813 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7814 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7815 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7816 %}
7817
7818 ins_pipe(pipe_class_default);
7819 %}
7820
7821 // Note: Long.bitCount(long) returns an int.
7822 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7823 match(Set dst (PopCountL src));
7824 effect(TEMP tmp);
7825 ins_cost(INSN_COST * 13);
7826
7827 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7828 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7829 "addv $tmp, $tmp\t# vector (8B)\n\t"
7830 "mov $dst, $tmp\t# vector (1D)" %}
7831 ins_encode %{
7832 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7833 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7834 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7835 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7836 %}
7837
7838 ins_pipe(pipe_class_default);
7839 %}
7840
7841 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7842 match(Set dst (PopCountL (LoadL mem)));
7843 effect(TEMP tmp);
7844 ins_cost(INSN_COST * 13);
7845
7846 format %{ "ldrd $tmp, $mem\n\t"
7847 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7848 "addv $tmp, $tmp\t# vector (8B)\n\t"
7849 "mov $dst, $tmp\t# vector (1D)" %}
7850 ins_encode %{
7851 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7852 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7853 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7854 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7855 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7856 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7857 %}
7858
7859 ins_pipe(pipe_class_default);
7860 %}
7861
7862 // ============================================================================
7863 // VerifyVectorAlignment Instruction
7864
7865 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7866 match(Set addr (VerifyVectorAlignment addr mask));
7867 effect(KILL cr);
7868 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7869 ins_encode %{
7870 Label Lskip;
7871 // check if masked bits of addr are zero
7872 __ tst($addr$$Register, $mask$$constant);
7873 __ br(Assembler::EQ, Lskip);
7874 __ stop("verify_vector_alignment found a misaligned vector memory access");
7875 __ bind(Lskip);
7876 %}
7877 ins_pipe(pipe_slow);
7878 %}
7879
7880 // ============================================================================
7881 // MemBar Instruction
7882
7883 instruct load_fence() %{
7884 match(LoadFence);
7885 ins_cost(VOLATILE_REF_COST);
7886
7887 format %{ "load_fence" %}
7888
7889 ins_encode %{
7890 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7891 %}
7892 ins_pipe(pipe_serial);
7893 %}
7894
7895 instruct unnecessary_membar_acquire() %{
7896 predicate(unnecessary_acquire(n));
7897 match(MemBarAcquire);
7898 ins_cost(0);
7899
7900 format %{ "membar_acquire (elided)" %}
7901
7902 ins_encode %{
7903 __ block_comment("membar_acquire (elided)");
7904 %}
7905
7906 ins_pipe(pipe_class_empty);
7907 %}
7908
7909 instruct membar_acquire() %{
7910 match(MemBarAcquire);
7911 ins_cost(VOLATILE_REF_COST);
7912
7913 format %{ "membar_acquire\n\t"
7914 "dmb ishld" %}
7915
7916 ins_encode %{
7917 __ block_comment("membar_acquire");
7918 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7919 %}
7920
7921 ins_pipe(pipe_serial);
7922 %}
7923
7924
7925 instruct membar_acquire_lock() %{
7926 match(MemBarAcquireLock);
7927 ins_cost(VOLATILE_REF_COST);
7928
7929 format %{ "membar_acquire_lock (elided)" %}
7930
7931 ins_encode %{
7932 __ block_comment("membar_acquire_lock (elided)");
7933 %}
7934
7935 ins_pipe(pipe_serial);
7936 %}
7937
7938 instruct store_fence() %{
7939 match(StoreFence);
7940 ins_cost(VOLATILE_REF_COST);
7941
7942 format %{ "store_fence" %}
7943
7944 ins_encode %{
7945 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7946 %}
7947 ins_pipe(pipe_serial);
7948 %}
7949
7950 instruct unnecessary_membar_release() %{
7951 predicate(unnecessary_release(n));
7952 match(MemBarRelease);
7953 ins_cost(0);
7954
7955 format %{ "membar_release (elided)" %}
7956
7957 ins_encode %{
7958 __ block_comment("membar_release (elided)");
7959 %}
7960 ins_pipe(pipe_serial);
7961 %}
7962
7963 instruct membar_release() %{
7964 match(MemBarRelease);
7965 ins_cost(VOLATILE_REF_COST);
7966
7967 format %{ "membar_release\n\t"
7968 "dmb ishst\n\tdmb ishld" %}
7969
7970 ins_encode %{
7971 __ block_comment("membar_release");
7972 // These will be merged if AlwaysMergeDMB is enabled.
7973 __ membar(Assembler::StoreStore);
7974 __ membar(Assembler::LoadStore);
7975 %}
7976 ins_pipe(pipe_serial);
7977 %}
7978
7979 instruct membar_storestore() %{
7980 match(MemBarStoreStore);
7981 match(StoreStoreFence);
7982 ins_cost(VOLATILE_REF_COST);
7983
7984 format %{ "MEMBAR-store-store" %}
7985
7986 ins_encode %{
7987 __ membar(Assembler::StoreStore);
7988 %}
7989 ins_pipe(pipe_serial);
7990 %}
7991
7992 instruct membar_release_lock() %{
7993 match(MemBarReleaseLock);
7994 ins_cost(VOLATILE_REF_COST);
7995
7996 format %{ "membar_release_lock (elided)" %}
7997
7998 ins_encode %{
7999 __ block_comment("membar_release_lock (elided)");
8000 %}
8001
8002 ins_pipe(pipe_serial);
8003 %}
8004
8005 instruct unnecessary_membar_volatile() %{
8006 predicate(unnecessary_volatile(n));
8007 match(MemBarVolatile);
8008 ins_cost(0);
8009
8010 format %{ "membar_volatile (elided)" %}
8011
8012 ins_encode %{
8013 __ block_comment("membar_volatile (elided)");
8014 %}
8015
8016 ins_pipe(pipe_serial);
8017 %}
8018
8019 instruct membar_volatile() %{
8020 match(MemBarVolatile);
8021 ins_cost(VOLATILE_REF_COST*100);
8022
8023 format %{ "membar_volatile\n\t"
8024 "dmb ish"%}
8025
8026 ins_encode %{
8027 __ block_comment("membar_volatile");
8028 __ membar(Assembler::StoreLoad);
8029 %}
8030
8031 ins_pipe(pipe_serial);
8032 %}
8033
8034 // ============================================================================
8035 // Cast/Convert Instructions
8036
8037 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8038 match(Set dst (CastX2P src));
8039
8040 ins_cost(INSN_COST);
8041 format %{ "mov $dst, $src\t# long -> ptr" %}
8042
8043 ins_encode %{
8044 if ($dst$$reg != $src$$reg) {
8045 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8046 }
8047 %}
8048
8049 ins_pipe(ialu_reg);
8050 %}
8051
8052 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8053 match(Set dst (CastP2X src));
8054
8055 ins_cost(INSN_COST);
8056 format %{ "mov $dst, $src\t# ptr -> long" %}
8057
8058 ins_encode %{
8059 if ($dst$$reg != $src$$reg) {
8060 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8061 }
8062 %}
8063
8064 ins_pipe(ialu_reg);
8065 %}
8066
8067 // Convert oop into int for vectors alignment masking
8068 instruct convP2I(iRegINoSp dst, iRegP src) %{
8069 match(Set dst (ConvL2I (CastP2X src)));
8070
8071 ins_cost(INSN_COST);
8072 format %{ "movw $dst, $src\t# ptr -> int" %}
8073 ins_encode %{
8074 __ movw($dst$$Register, $src$$Register);
8075 %}
8076
8077 ins_pipe(ialu_reg);
8078 %}
8079
8080 // Convert compressed oop into int for vectors alignment masking
8081 // in case of 32bit oops (heap < 4Gb).
8082 instruct convN2I(iRegINoSp dst, iRegN src)
8083 %{
8084 predicate(CompressedOops::shift() == 0);
8085 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8086
8087 ins_cost(INSN_COST);
8088 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8089 ins_encode %{
8090 __ movw($dst$$Register, $src$$Register);
8091 %}
8092
8093 ins_pipe(ialu_reg);
8094 %}
8095
8096
8097 // Convert oop pointer into compressed form
8098 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8099 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8100 match(Set dst (EncodeP src));
8101 effect(KILL cr);
8102 ins_cost(INSN_COST * 3);
8103 format %{ "encode_heap_oop $dst, $src" %}
8104 ins_encode %{
8105 Register s = $src$$Register;
8106 Register d = $dst$$Register;
8107 __ encode_heap_oop(d, s);
8108 %}
8109 ins_pipe(ialu_reg);
8110 %}
8111
8112 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8113 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8114 match(Set dst (EncodeP src));
8115 ins_cost(INSN_COST * 3);
8116 format %{ "encode_heap_oop_not_null $dst, $src" %}
8117 ins_encode %{
8118 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8119 %}
8120 ins_pipe(ialu_reg);
8121 %}
8122
8123 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8124 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8125 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8126 match(Set dst (DecodeN src));
8127 ins_cost(INSN_COST * 3);
8128 format %{ "decode_heap_oop $dst, $src" %}
8129 ins_encode %{
8130 Register s = $src$$Register;
8131 Register d = $dst$$Register;
8132 __ decode_heap_oop(d, s);
8133 %}
8134 ins_pipe(ialu_reg);
8135 %}
8136
8137 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8138 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8139 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8140 match(Set dst (DecodeN src));
8141 ins_cost(INSN_COST * 3);
8142 format %{ "decode_heap_oop_not_null $dst, $src" %}
8143 ins_encode %{
8144 Register s = $src$$Register;
8145 Register d = $dst$$Register;
8146 __ decode_heap_oop_not_null(d, s);
8147 %}
8148 ins_pipe(ialu_reg);
8149 %}
8150
8151 // n.b. AArch64 implementations of encode_klass_not_null and
8152 // decode_klass_not_null do not modify the flags register so, unlike
8153 // Intel, we don't kill CR as a side effect here
8154
8155 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8156 match(Set dst (EncodePKlass src));
8157
8158 ins_cost(INSN_COST * 3);
8159 format %{ "encode_klass_not_null $dst,$src" %}
8160
8161 ins_encode %{
8162 Register src_reg = as_Register($src$$reg);
8163 Register dst_reg = as_Register($dst$$reg);
8164 __ encode_klass_not_null(dst_reg, src_reg);
8165 %}
8166
8167 ins_pipe(ialu_reg);
8168 %}
8169
8170 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8171 match(Set dst (DecodeNKlass src));
8172
8173 ins_cost(INSN_COST * 3);
8174 format %{ "decode_klass_not_null $dst,$src" %}
8175
8176 ins_encode %{
8177 Register src_reg = as_Register($src$$reg);
8178 Register dst_reg = as_Register($dst$$reg);
8179 if (dst_reg != src_reg) {
8180 __ decode_klass_not_null(dst_reg, src_reg);
8181 } else {
8182 __ decode_klass_not_null(dst_reg);
8183 }
8184 %}
8185
8186 ins_pipe(ialu_reg);
8187 %}
8188
8189 instruct checkCastPP(iRegPNoSp dst)
8190 %{
8191 match(Set dst (CheckCastPP dst));
8192
8193 size(0);
8194 format %{ "# checkcastPP of $dst" %}
8195 ins_encode(/* empty encoding */);
8196 ins_pipe(pipe_class_empty);
8197 %}
8198
8199 instruct castPP(iRegPNoSp dst)
8200 %{
8201 match(Set dst (CastPP dst));
8202
8203 size(0);
8204 format %{ "# castPP of $dst" %}
8205 ins_encode(/* empty encoding */);
8206 ins_pipe(pipe_class_empty);
8207 %}
8208
8209 instruct castII(iRegI dst)
8210 %{
8211 predicate(VerifyConstraintCasts == 0);
8212 match(Set dst (CastII dst));
8213
8214 size(0);
8215 format %{ "# castII of $dst" %}
8216 ins_encode(/* empty encoding */);
8217 ins_cost(0);
8218 ins_pipe(pipe_class_empty);
8219 %}
8220
8221 instruct castII_checked(iRegI dst, rFlagsReg cr)
8222 %{
8223 predicate(VerifyConstraintCasts > 0);
8224 match(Set dst (CastII dst));
8225 effect(KILL cr);
8226
8227 format %{ "# castII_checked of $dst" %}
8228 ins_encode %{
8229 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8230 %}
8231 ins_pipe(pipe_slow);
8232 %}
8233
8234 instruct castLL(iRegL dst)
8235 %{
8236 predicate(VerifyConstraintCasts == 0);
8237 match(Set dst (CastLL dst));
8238
8239 size(0);
8240 format %{ "# castLL of $dst" %}
8241 ins_encode(/* empty encoding */);
8242 ins_cost(0);
8243 ins_pipe(pipe_class_empty);
8244 %}
8245
8246 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8247 %{
8248 predicate(VerifyConstraintCasts > 0);
8249 match(Set dst (CastLL dst));
8250 effect(KILL cr);
8251
8252 format %{ "# castLL_checked of $dst" %}
8253 ins_encode %{
8254 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8255 %}
8256 ins_pipe(pipe_slow);
8257 %}
8258
8259 instruct castHH(vRegF dst)
8260 %{
8261 match(Set dst (CastHH dst));
8262 size(0);
8263 format %{ "# castHH of $dst" %}
8264 ins_encode(/* empty encoding */);
8265 ins_cost(0);
8266 ins_pipe(pipe_class_empty);
8267 %}
8268
8269 instruct castFF(vRegF dst)
8270 %{
8271 match(Set dst (CastFF dst));
8272
8273 size(0);
8274 format %{ "# castFF of $dst" %}
8275 ins_encode(/* empty encoding */);
8276 ins_cost(0);
8277 ins_pipe(pipe_class_empty);
8278 %}
8279
8280 instruct castDD(vRegD dst)
8281 %{
8282 match(Set dst (CastDD dst));
8283
8284 size(0);
8285 format %{ "# castDD of $dst" %}
8286 ins_encode(/* empty encoding */);
8287 ins_cost(0);
8288 ins_pipe(pipe_class_empty);
8289 %}
8290
8291 instruct castVV(vReg dst)
8292 %{
8293 match(Set dst (CastVV dst));
8294
8295 size(0);
8296 format %{ "# castVV of $dst" %}
8297 ins_encode(/* empty encoding */);
8298 ins_cost(0);
8299 ins_pipe(pipe_class_empty);
8300 %}
8301
8302 instruct castVVMask(pRegGov dst)
8303 %{
8304 match(Set dst (CastVV dst));
8305
8306 size(0);
8307 format %{ "# castVV of $dst" %}
8308 ins_encode(/* empty encoding */);
8309 ins_cost(0);
8310 ins_pipe(pipe_class_empty);
8311 %}
8312
8313 // Manifest a CmpU result in an integer register.
8314 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8315 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8316 %{
8317 match(Set dst (CmpU3 src1 src2));
8318 effect(KILL flags);
8319
8320 ins_cost(INSN_COST * 3);
8321 format %{
8322 "cmpw $src1, $src2\n\t"
8323 "csetw $dst, ne\n\t"
8324 "cnegw $dst, lo\t# CmpU3(reg)"
8325 %}
8326 ins_encode %{
8327 __ cmpw($src1$$Register, $src2$$Register);
8328 __ csetw($dst$$Register, Assembler::NE);
8329 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8330 %}
8331
8332 ins_pipe(pipe_class_default);
8333 %}
8334
8335 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8336 %{
8337 match(Set dst (CmpU3 src1 src2));
8338 effect(KILL flags);
8339
8340 ins_cost(INSN_COST * 3);
8341 format %{
8342 "subsw zr, $src1, $src2\n\t"
8343 "csetw $dst, ne\n\t"
8344 "cnegw $dst, lo\t# CmpU3(imm)"
8345 %}
8346 ins_encode %{
8347 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8348 __ csetw($dst$$Register, Assembler::NE);
8349 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8350 %}
8351
8352 ins_pipe(pipe_class_default);
8353 %}
8354
8355 // Manifest a CmpUL result in an integer register.
8356 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8357 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8358 %{
8359 match(Set dst (CmpUL3 src1 src2));
8360 effect(KILL flags);
8361
8362 ins_cost(INSN_COST * 3);
8363 format %{
8364 "cmp $src1, $src2\n\t"
8365 "csetw $dst, ne\n\t"
8366 "cnegw $dst, lo\t# CmpUL3(reg)"
8367 %}
8368 ins_encode %{
8369 __ cmp($src1$$Register, $src2$$Register);
8370 __ csetw($dst$$Register, Assembler::NE);
8371 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8372 %}
8373
8374 ins_pipe(pipe_class_default);
8375 %}
8376
8377 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8378 %{
8379 match(Set dst (CmpUL3 src1 src2));
8380 effect(KILL flags);
8381
8382 ins_cost(INSN_COST * 3);
8383 format %{
8384 "subs zr, $src1, $src2\n\t"
8385 "csetw $dst, ne\n\t"
8386 "cnegw $dst, lo\t# CmpUL3(imm)"
8387 %}
8388 ins_encode %{
8389 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8390 __ csetw($dst$$Register, Assembler::NE);
8391 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8392 %}
8393
8394 ins_pipe(pipe_class_default);
8395 %}
8396
8397 // Manifest a CmpL result in an integer register.
8398 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8399 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8400 %{
8401 match(Set dst (CmpL3 src1 src2));
8402 effect(KILL flags);
8403
8404 ins_cost(INSN_COST * 3);
8405 format %{
8406 "cmp $src1, $src2\n\t"
8407 "csetw $dst, ne\n\t"
8408 "cnegw $dst, lt\t# CmpL3(reg)"
8409 %}
8410 ins_encode %{
8411 __ cmp($src1$$Register, $src2$$Register);
8412 __ csetw($dst$$Register, Assembler::NE);
8413 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8414 %}
8415
8416 ins_pipe(pipe_class_default);
8417 %}
8418
8419 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8420 %{
8421 match(Set dst (CmpL3 src1 src2));
8422 effect(KILL flags);
8423
8424 ins_cost(INSN_COST * 3);
8425 format %{
8426 "subs zr, $src1, $src2\n\t"
8427 "csetw $dst, ne\n\t"
8428 "cnegw $dst, lt\t# CmpL3(imm)"
8429 %}
8430 ins_encode %{
8431 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8432 __ csetw($dst$$Register, Assembler::NE);
8433 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8434 %}
8435
8436 ins_pipe(pipe_class_default);
8437 %}
8438
8439 // ============================================================================
8440 // Conditional Move Instructions
8441
8442 // n.b. we have identical rules for both a signed compare op (cmpOp)
8443 // and an unsigned compare op (cmpOpU). it would be nice if we could
8444 // define an op class which merged both inputs and use it to type the
8445 // argument to a single rule. unfortunatelyt his fails because the
8446 // opclass does not live up to the COND_INTER interface of its
8447 // component operands. When the generic code tries to negate the
8448 // operand it ends up running the generci Machoper::negate method
8449 // which throws a ShouldNotHappen. So, we have to provide two flavours
8450 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8451
8452 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8453 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8454
8455 ins_cost(INSN_COST * 2);
8456 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8457
8458 ins_encode %{
8459 __ cselw(as_Register($dst$$reg),
8460 as_Register($src2$$reg),
8461 as_Register($src1$$reg),
8462 (Assembler::Condition)$cmp$$cmpcode);
8463 %}
8464
8465 ins_pipe(icond_reg_reg);
8466 %}
8467
8468 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8469 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8470
8471 ins_cost(INSN_COST * 2);
8472 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8473
8474 ins_encode %{
8475 __ cselw(as_Register($dst$$reg),
8476 as_Register($src2$$reg),
8477 as_Register($src1$$reg),
8478 (Assembler::Condition)$cmp$$cmpcode);
8479 %}
8480
8481 ins_pipe(icond_reg_reg);
8482 %}
8483
8484 // special cases where one arg is zero
8485
8486 // n.b. this is selected in preference to the rule above because it
8487 // avoids loading constant 0 into a source register
8488
8489 // TODO
8490 // we ought only to be able to cull one of these variants as the ideal
8491 // transforms ought always to order the zero consistently (to left/right?)
8492
8493 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8494 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8495
8496 ins_cost(INSN_COST * 2);
8497 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8498
8499 ins_encode %{
8500 __ cselw(as_Register($dst$$reg),
8501 as_Register($src$$reg),
8502 zr,
8503 (Assembler::Condition)$cmp$$cmpcode);
8504 %}
8505
8506 ins_pipe(icond_reg);
8507 %}
8508
8509 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8510 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8511
8512 ins_cost(INSN_COST * 2);
8513 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8514
8515 ins_encode %{
8516 __ cselw(as_Register($dst$$reg),
8517 as_Register($src$$reg),
8518 zr,
8519 (Assembler::Condition)$cmp$$cmpcode);
8520 %}
8521
8522 ins_pipe(icond_reg);
8523 %}
8524
8525 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8526 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8527
8528 ins_cost(INSN_COST * 2);
8529 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8530
8531 ins_encode %{
8532 __ cselw(as_Register($dst$$reg),
8533 zr,
8534 as_Register($src$$reg),
8535 (Assembler::Condition)$cmp$$cmpcode);
8536 %}
8537
8538 ins_pipe(icond_reg);
8539 %}
8540
8541 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8542 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8543
8544 ins_cost(INSN_COST * 2);
8545 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8546
8547 ins_encode %{
8548 __ cselw(as_Register($dst$$reg),
8549 zr,
8550 as_Register($src$$reg),
8551 (Assembler::Condition)$cmp$$cmpcode);
8552 %}
8553
8554 ins_pipe(icond_reg);
8555 %}
8556
8557 // special case for creating a boolean 0 or 1
8558
8559 // n.b. this is selected in preference to the rule above because it
8560 // avoids loading constants 0 and 1 into a source register
8561
8562 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8563 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8564
8565 ins_cost(INSN_COST * 2);
8566 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8567
8568 ins_encode %{
8569 // equivalently
8570 // cset(as_Register($dst$$reg),
8571 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8572 __ csincw(as_Register($dst$$reg),
8573 zr,
8574 zr,
8575 (Assembler::Condition)$cmp$$cmpcode);
8576 %}
8577
8578 ins_pipe(icond_none);
8579 %}
8580
8581 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8582 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8583
8584 ins_cost(INSN_COST * 2);
8585 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8586
8587 ins_encode %{
8588 // equivalently
8589 // cset(as_Register($dst$$reg),
8590 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8591 __ csincw(as_Register($dst$$reg),
8592 zr,
8593 zr,
8594 (Assembler::Condition)$cmp$$cmpcode);
8595 %}
8596
8597 ins_pipe(icond_none);
8598 %}
8599
8600 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8601 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8602
8603 ins_cost(INSN_COST * 2);
8604 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8605
8606 ins_encode %{
8607 __ csel(as_Register($dst$$reg),
8608 as_Register($src2$$reg),
8609 as_Register($src1$$reg),
8610 (Assembler::Condition)$cmp$$cmpcode);
8611 %}
8612
8613 ins_pipe(icond_reg_reg);
8614 %}
8615
8616 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8617 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8618
8619 ins_cost(INSN_COST * 2);
8620 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8621
8622 ins_encode %{
8623 __ csel(as_Register($dst$$reg),
8624 as_Register($src2$$reg),
8625 as_Register($src1$$reg),
8626 (Assembler::Condition)$cmp$$cmpcode);
8627 %}
8628
8629 ins_pipe(icond_reg_reg);
8630 %}
8631
8632 // special cases where one arg is zero
8633
8634 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8635 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8636
8637 ins_cost(INSN_COST * 2);
8638 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8639
8640 ins_encode %{
8641 __ csel(as_Register($dst$$reg),
8642 zr,
8643 as_Register($src$$reg),
8644 (Assembler::Condition)$cmp$$cmpcode);
8645 %}
8646
8647 ins_pipe(icond_reg);
8648 %}
8649
8650 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8651 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8652
8653 ins_cost(INSN_COST * 2);
8654 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8655
8656 ins_encode %{
8657 __ csel(as_Register($dst$$reg),
8658 zr,
8659 as_Register($src$$reg),
8660 (Assembler::Condition)$cmp$$cmpcode);
8661 %}
8662
8663 ins_pipe(icond_reg);
8664 %}
8665
8666 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8667 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8668
8669 ins_cost(INSN_COST * 2);
8670 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8671
8672 ins_encode %{
8673 __ csel(as_Register($dst$$reg),
8674 as_Register($src$$reg),
8675 zr,
8676 (Assembler::Condition)$cmp$$cmpcode);
8677 %}
8678
8679 ins_pipe(icond_reg);
8680 %}
8681
8682 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8683 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8684
8685 ins_cost(INSN_COST * 2);
8686 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8687
8688 ins_encode %{
8689 __ csel(as_Register($dst$$reg),
8690 as_Register($src$$reg),
8691 zr,
8692 (Assembler::Condition)$cmp$$cmpcode);
8693 %}
8694
8695 ins_pipe(icond_reg);
8696 %}
8697
8698 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8699 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8700
8701 ins_cost(INSN_COST * 2);
8702 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8703
8704 ins_encode %{
8705 __ csel(as_Register($dst$$reg),
8706 as_Register($src2$$reg),
8707 as_Register($src1$$reg),
8708 (Assembler::Condition)$cmp$$cmpcode);
8709 %}
8710
8711 ins_pipe(icond_reg_reg);
8712 %}
8713
8714 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8715 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8716
8717 ins_cost(INSN_COST * 2);
8718 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8719
8720 ins_encode %{
8721 __ csel(as_Register($dst$$reg),
8722 as_Register($src2$$reg),
8723 as_Register($src1$$reg),
8724 (Assembler::Condition)$cmp$$cmpcode);
8725 %}
8726
8727 ins_pipe(icond_reg_reg);
8728 %}
8729
8730 // special cases where one arg is zero
8731
8732 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8733 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8734
8735 ins_cost(INSN_COST * 2);
8736 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8737
8738 ins_encode %{
8739 __ csel(as_Register($dst$$reg),
8740 zr,
8741 as_Register($src$$reg),
8742 (Assembler::Condition)$cmp$$cmpcode);
8743 %}
8744
8745 ins_pipe(icond_reg);
8746 %}
8747
8748 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8749 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8750
8751 ins_cost(INSN_COST * 2);
8752 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8753
8754 ins_encode %{
8755 __ csel(as_Register($dst$$reg),
8756 zr,
8757 as_Register($src$$reg),
8758 (Assembler::Condition)$cmp$$cmpcode);
8759 %}
8760
8761 ins_pipe(icond_reg);
8762 %}
8763
8764 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8765 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8766
8767 ins_cost(INSN_COST * 2);
8768 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8769
8770 ins_encode %{
8771 __ csel(as_Register($dst$$reg),
8772 as_Register($src$$reg),
8773 zr,
8774 (Assembler::Condition)$cmp$$cmpcode);
8775 %}
8776
8777 ins_pipe(icond_reg);
8778 %}
8779
8780 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8781 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8782
8783 ins_cost(INSN_COST * 2);
8784 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8785
8786 ins_encode %{
8787 __ csel(as_Register($dst$$reg),
8788 as_Register($src$$reg),
8789 zr,
8790 (Assembler::Condition)$cmp$$cmpcode);
8791 %}
8792
8793 ins_pipe(icond_reg);
8794 %}
8795
8796 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8797 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8798
8799 ins_cost(INSN_COST * 2);
8800 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8801
8802 ins_encode %{
8803 __ cselw(as_Register($dst$$reg),
8804 as_Register($src2$$reg),
8805 as_Register($src1$$reg),
8806 (Assembler::Condition)$cmp$$cmpcode);
8807 %}
8808
8809 ins_pipe(icond_reg_reg);
8810 %}
8811
8812 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8813 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8814
8815 ins_cost(INSN_COST * 2);
8816 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8817
8818 ins_encode %{
8819 __ cselw(as_Register($dst$$reg),
8820 as_Register($src2$$reg),
8821 as_Register($src1$$reg),
8822 (Assembler::Condition)$cmp$$cmpcode);
8823 %}
8824
8825 ins_pipe(icond_reg_reg);
8826 %}
8827
8828 // special cases where one arg is zero
8829
8830 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8831 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8832
8833 ins_cost(INSN_COST * 2);
8834 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8835
8836 ins_encode %{
8837 __ cselw(as_Register($dst$$reg),
8838 zr,
8839 as_Register($src$$reg),
8840 (Assembler::Condition)$cmp$$cmpcode);
8841 %}
8842
8843 ins_pipe(icond_reg);
8844 %}
8845
8846 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8847 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8848
8849 ins_cost(INSN_COST * 2);
8850 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8851
8852 ins_encode %{
8853 __ cselw(as_Register($dst$$reg),
8854 zr,
8855 as_Register($src$$reg),
8856 (Assembler::Condition)$cmp$$cmpcode);
8857 %}
8858
8859 ins_pipe(icond_reg);
8860 %}
8861
8862 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8863 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8864
8865 ins_cost(INSN_COST * 2);
8866 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8867
8868 ins_encode %{
8869 __ cselw(as_Register($dst$$reg),
8870 as_Register($src$$reg),
8871 zr,
8872 (Assembler::Condition)$cmp$$cmpcode);
8873 %}
8874
8875 ins_pipe(icond_reg);
8876 %}
8877
8878 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8879 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8880
8881 ins_cost(INSN_COST * 2);
8882 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8883
8884 ins_encode %{
8885 __ cselw(as_Register($dst$$reg),
8886 as_Register($src$$reg),
8887 zr,
8888 (Assembler::Condition)$cmp$$cmpcode);
8889 %}
8890
8891 ins_pipe(icond_reg);
8892 %}
8893
8894 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8895 %{
8896 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8897
8898 ins_cost(INSN_COST * 3);
8899
8900 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8901 ins_encode %{
8902 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8903 __ fcsels(as_FloatRegister($dst$$reg),
8904 as_FloatRegister($src2$$reg),
8905 as_FloatRegister($src1$$reg),
8906 cond);
8907 %}
8908
8909 ins_pipe(fp_cond_reg_reg_s);
8910 %}
8911
8912 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8913 %{
8914 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8915
8916 ins_cost(INSN_COST * 3);
8917
8918 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8919 ins_encode %{
8920 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8921 __ fcsels(as_FloatRegister($dst$$reg),
8922 as_FloatRegister($src2$$reg),
8923 as_FloatRegister($src1$$reg),
8924 cond);
8925 %}
8926
8927 ins_pipe(fp_cond_reg_reg_s);
8928 %}
8929
8930 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8931 %{
8932 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8933
8934 ins_cost(INSN_COST * 3);
8935
8936 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8937 ins_encode %{
8938 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8939 __ fcseld(as_FloatRegister($dst$$reg),
8940 as_FloatRegister($src2$$reg),
8941 as_FloatRegister($src1$$reg),
8942 cond);
8943 %}
8944
8945 ins_pipe(fp_cond_reg_reg_d);
8946 %}
8947
8948 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
8949 %{
8950 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8951
8952 ins_cost(INSN_COST * 3);
8953
8954 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8955 ins_encode %{
8956 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8957 __ fcseld(as_FloatRegister($dst$$reg),
8958 as_FloatRegister($src2$$reg),
8959 as_FloatRegister($src1$$reg),
8960 cond);
8961 %}
8962
8963 ins_pipe(fp_cond_reg_reg_d);
8964 %}
8965
8966 // ============================================================================
8967 // Arithmetic Instructions
8968 //
8969
8970 // Integer Addition
8971
8972 // TODO
8973 // these currently employ operations which do not set CR and hence are
8974 // not flagged as killing CR but we would like to isolate the cases
8975 // where we want to set flags from those where we don't. need to work
8976 // out how to do that.
8977
8978 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8979 match(Set dst (AddI src1 src2));
8980
8981 ins_cost(INSN_COST);
8982 format %{ "addw $dst, $src1, $src2" %}
8983
8984 ins_encode %{
8985 __ addw(as_Register($dst$$reg),
8986 as_Register($src1$$reg),
8987 as_Register($src2$$reg));
8988 %}
8989
8990 ins_pipe(ialu_reg_reg);
8991 %}
8992
8993 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
8994 match(Set dst (AddI src1 src2));
8995
8996 ins_cost(INSN_COST);
8997 format %{ "addw $dst, $src1, $src2" %}
8998
8999 // use opcode to indicate that this is an add not a sub
9000 opcode(0x0);
9001
9002 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9003
9004 ins_pipe(ialu_reg_imm);
9005 %}
9006
9007 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9008 match(Set dst (AddI (ConvL2I src1) src2));
9009
9010 ins_cost(INSN_COST);
9011 format %{ "addw $dst, $src1, $src2" %}
9012
9013 // use opcode to indicate that this is an add not a sub
9014 opcode(0x0);
9015
9016 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9017
9018 ins_pipe(ialu_reg_imm);
9019 %}
9020
9021 // Pointer Addition
9022 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9023 match(Set dst (AddP src1 src2));
9024
9025 ins_cost(INSN_COST);
9026 format %{ "add $dst, $src1, $src2\t# ptr" %}
9027
9028 ins_encode %{
9029 __ add(as_Register($dst$$reg),
9030 as_Register($src1$$reg),
9031 as_Register($src2$$reg));
9032 %}
9033
9034 ins_pipe(ialu_reg_reg);
9035 %}
9036
9037 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9038 match(Set dst (AddP src1 (ConvI2L src2)));
9039
9040 ins_cost(1.9 * INSN_COST);
9041 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9042
9043 ins_encode %{
9044 __ add(as_Register($dst$$reg),
9045 as_Register($src1$$reg),
9046 as_Register($src2$$reg), ext::sxtw);
9047 %}
9048
9049 ins_pipe(ialu_reg_reg);
9050 %}
9051
9052 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9053 match(Set dst (AddP src1 (LShiftL src2 scale)));
9054
9055 ins_cost(1.9 * INSN_COST);
9056 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9057
9058 ins_encode %{
9059 __ lea(as_Register($dst$$reg),
9060 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9061 Address::lsl($scale$$constant)));
9062 %}
9063
9064 ins_pipe(ialu_reg_reg_shift);
9065 %}
9066
9067 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9068 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9069
9070 ins_cost(1.9 * INSN_COST);
9071 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9072
9073 ins_encode %{
9074 __ lea(as_Register($dst$$reg),
9075 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9076 Address::sxtw($scale$$constant)));
9077 %}
9078
9079 ins_pipe(ialu_reg_reg_shift);
9080 %}
9081
9082 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9083 match(Set dst (LShiftL (ConvI2L src) scale));
9084
9085 ins_cost(INSN_COST);
9086 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9087
9088 ins_encode %{
9089 __ sbfiz(as_Register($dst$$reg),
9090 as_Register($src$$reg),
9091 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9092 %}
9093
9094 ins_pipe(ialu_reg_shift);
9095 %}
9096
9097 // Pointer Immediate Addition
9098 // n.b. this needs to be more expensive than using an indirect memory
9099 // operand
9100 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9101 match(Set dst (AddP src1 src2));
9102
9103 ins_cost(INSN_COST);
9104 format %{ "add $dst, $src1, $src2\t# ptr" %}
9105
9106 // use opcode to indicate that this is an add not a sub
9107 opcode(0x0);
9108
9109 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9110
9111 ins_pipe(ialu_reg_imm);
9112 %}
9113
9114 // Long Addition
9115 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9116
9117 match(Set dst (AddL src1 src2));
9118
9119 ins_cost(INSN_COST);
9120 format %{ "add $dst, $src1, $src2" %}
9121
9122 ins_encode %{
9123 __ add(as_Register($dst$$reg),
9124 as_Register($src1$$reg),
9125 as_Register($src2$$reg));
9126 %}
9127
9128 ins_pipe(ialu_reg_reg);
9129 %}
9130
9131 // No constant pool entries requiredLong Immediate Addition.
9132 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9133 match(Set dst (AddL src1 src2));
9134
9135 ins_cost(INSN_COST);
9136 format %{ "add $dst, $src1, $src2" %}
9137
9138 // use opcode to indicate that this is an add not a sub
9139 opcode(0x0);
9140
9141 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9142
9143 ins_pipe(ialu_reg_imm);
9144 %}
9145
9146 // Integer Subtraction
9147 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9148 match(Set dst (SubI src1 src2));
9149
9150 ins_cost(INSN_COST);
9151 format %{ "subw $dst, $src1, $src2" %}
9152
9153 ins_encode %{
9154 __ subw(as_Register($dst$$reg),
9155 as_Register($src1$$reg),
9156 as_Register($src2$$reg));
9157 %}
9158
9159 ins_pipe(ialu_reg_reg);
9160 %}
9161
9162 // Immediate Subtraction
9163 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9164 match(Set dst (SubI src1 src2));
9165
9166 ins_cost(INSN_COST);
9167 format %{ "subw $dst, $src1, $src2" %}
9168
9169 // use opcode to indicate that this is a sub not an add
9170 opcode(0x1);
9171
9172 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9173
9174 ins_pipe(ialu_reg_imm);
9175 %}
9176
9177 // Long Subtraction
9178 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9179
9180 match(Set dst (SubL src1 src2));
9181
9182 ins_cost(INSN_COST);
9183 format %{ "sub $dst, $src1, $src2" %}
9184
9185 ins_encode %{
9186 __ sub(as_Register($dst$$reg),
9187 as_Register($src1$$reg),
9188 as_Register($src2$$reg));
9189 %}
9190
9191 ins_pipe(ialu_reg_reg);
9192 %}
9193
9194 // No constant pool entries requiredLong Immediate Subtraction.
9195 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9196 match(Set dst (SubL src1 src2));
9197
9198 ins_cost(INSN_COST);
9199 format %{ "sub$dst, $src1, $src2" %}
9200
9201 // use opcode to indicate that this is a sub not an add
9202 opcode(0x1);
9203
9204 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9205
9206 ins_pipe(ialu_reg_imm);
9207 %}
9208
9209 // Integer Negation (special case for sub)
9210
9211 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9212 match(Set dst (SubI zero src));
9213
9214 ins_cost(INSN_COST);
9215 format %{ "negw $dst, $src\t# int" %}
9216
9217 ins_encode %{
9218 __ negw(as_Register($dst$$reg),
9219 as_Register($src$$reg));
9220 %}
9221
9222 ins_pipe(ialu_reg);
9223 %}
9224
9225 // Long Negation
9226
9227 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9228 match(Set dst (SubL zero src));
9229
9230 ins_cost(INSN_COST);
9231 format %{ "neg $dst, $src\t# long" %}
9232
9233 ins_encode %{
9234 __ neg(as_Register($dst$$reg),
9235 as_Register($src$$reg));
9236 %}
9237
9238 ins_pipe(ialu_reg);
9239 %}
9240
9241 // Integer Multiply
9242
9243 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9244 match(Set dst (MulI src1 src2));
9245
9246 ins_cost(INSN_COST * 3);
9247 format %{ "mulw $dst, $src1, $src2" %}
9248
9249 ins_encode %{
9250 __ mulw(as_Register($dst$$reg),
9251 as_Register($src1$$reg),
9252 as_Register($src2$$reg));
9253 %}
9254
9255 ins_pipe(imul_reg_reg);
9256 %}
9257
9258 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9259 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9260
9261 ins_cost(INSN_COST * 3);
9262 format %{ "smull $dst, $src1, $src2" %}
9263
9264 ins_encode %{
9265 __ smull(as_Register($dst$$reg),
9266 as_Register($src1$$reg),
9267 as_Register($src2$$reg));
9268 %}
9269
9270 ins_pipe(imul_reg_reg);
9271 %}
9272
9273 // Long Multiply
9274
9275 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9276 match(Set dst (MulL src1 src2));
9277
9278 ins_cost(INSN_COST * 5);
9279 format %{ "mul $dst, $src1, $src2" %}
9280
9281 ins_encode %{
9282 __ mul(as_Register($dst$$reg),
9283 as_Register($src1$$reg),
9284 as_Register($src2$$reg));
9285 %}
9286
9287 ins_pipe(lmul_reg_reg);
9288 %}
9289
9290 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9291 %{
9292 match(Set dst (MulHiL src1 src2));
9293
9294 ins_cost(INSN_COST * 7);
9295 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9296
9297 ins_encode %{
9298 __ smulh(as_Register($dst$$reg),
9299 as_Register($src1$$reg),
9300 as_Register($src2$$reg));
9301 %}
9302
9303 ins_pipe(lmul_reg_reg);
9304 %}
9305
9306 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9307 %{
9308 match(Set dst (UMulHiL src1 src2));
9309
9310 ins_cost(INSN_COST * 7);
9311 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9312
9313 ins_encode %{
9314 __ umulh(as_Register($dst$$reg),
9315 as_Register($src1$$reg),
9316 as_Register($src2$$reg));
9317 %}
9318
9319 ins_pipe(lmul_reg_reg);
9320 %}
9321
9322 // Combined Integer Multiply & Add/Sub
9323
9324 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9325 match(Set dst (AddI src3 (MulI src1 src2)));
9326
9327 ins_cost(INSN_COST * 3);
9328 format %{ "madd $dst, $src1, $src2, $src3" %}
9329
9330 ins_encode %{
9331 __ maddw(as_Register($dst$$reg),
9332 as_Register($src1$$reg),
9333 as_Register($src2$$reg),
9334 as_Register($src3$$reg));
9335 %}
9336
9337 ins_pipe(imac_reg_reg);
9338 %}
9339
9340 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9341 match(Set dst (SubI src3 (MulI src1 src2)));
9342
9343 ins_cost(INSN_COST * 3);
9344 format %{ "msub $dst, $src1, $src2, $src3" %}
9345
9346 ins_encode %{
9347 __ msubw(as_Register($dst$$reg),
9348 as_Register($src1$$reg),
9349 as_Register($src2$$reg),
9350 as_Register($src3$$reg));
9351 %}
9352
9353 ins_pipe(imac_reg_reg);
9354 %}
9355
9356 // Combined Integer Multiply & Neg
9357
9358 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9359 match(Set dst (MulI (SubI zero src1) src2));
9360
9361 ins_cost(INSN_COST * 3);
9362 format %{ "mneg $dst, $src1, $src2" %}
9363
9364 ins_encode %{
9365 __ mnegw(as_Register($dst$$reg),
9366 as_Register($src1$$reg),
9367 as_Register($src2$$reg));
9368 %}
9369
9370 ins_pipe(imac_reg_reg);
9371 %}
9372
9373 // Combined Long Multiply & Add/Sub
9374
9375 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9376 match(Set dst (AddL src3 (MulL src1 src2)));
9377
9378 ins_cost(INSN_COST * 5);
9379 format %{ "madd $dst, $src1, $src2, $src3" %}
9380
9381 ins_encode %{
9382 __ madd(as_Register($dst$$reg),
9383 as_Register($src1$$reg),
9384 as_Register($src2$$reg),
9385 as_Register($src3$$reg));
9386 %}
9387
9388 ins_pipe(lmac_reg_reg);
9389 %}
9390
9391 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9392 match(Set dst (SubL src3 (MulL src1 src2)));
9393
9394 ins_cost(INSN_COST * 5);
9395 format %{ "msub $dst, $src1, $src2, $src3" %}
9396
9397 ins_encode %{
9398 __ msub(as_Register($dst$$reg),
9399 as_Register($src1$$reg),
9400 as_Register($src2$$reg),
9401 as_Register($src3$$reg));
9402 %}
9403
9404 ins_pipe(lmac_reg_reg);
9405 %}
9406
9407 // Combined Long Multiply & Neg
9408
9409 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9410 match(Set dst (MulL (SubL zero src1) src2));
9411
9412 ins_cost(INSN_COST * 5);
9413 format %{ "mneg $dst, $src1, $src2" %}
9414
9415 ins_encode %{
9416 __ mneg(as_Register($dst$$reg),
9417 as_Register($src1$$reg),
9418 as_Register($src2$$reg));
9419 %}
9420
9421 ins_pipe(lmac_reg_reg);
9422 %}
9423
9424 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9425
9426 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9427 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9428
9429 ins_cost(INSN_COST * 3);
9430 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9431
9432 ins_encode %{
9433 __ smaddl(as_Register($dst$$reg),
9434 as_Register($src1$$reg),
9435 as_Register($src2$$reg),
9436 as_Register($src3$$reg));
9437 %}
9438
9439 ins_pipe(imac_reg_reg);
9440 %}
9441
9442 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9443 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9444
9445 ins_cost(INSN_COST * 3);
9446 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9447
9448 ins_encode %{
9449 __ smsubl(as_Register($dst$$reg),
9450 as_Register($src1$$reg),
9451 as_Register($src2$$reg),
9452 as_Register($src3$$reg));
9453 %}
9454
9455 ins_pipe(imac_reg_reg);
9456 %}
9457
9458 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9459 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9460
9461 ins_cost(INSN_COST * 3);
9462 format %{ "smnegl $dst, $src1, $src2" %}
9463
9464 ins_encode %{
9465 __ smnegl(as_Register($dst$$reg),
9466 as_Register($src1$$reg),
9467 as_Register($src2$$reg));
9468 %}
9469
9470 ins_pipe(imac_reg_reg);
9471 %}
9472
9473 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9474
9475 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9476 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9477
9478 ins_cost(INSN_COST * 5);
9479 format %{ "mulw rscratch1, $src1, $src2\n\t"
9480 "maddw $dst, $src3, $src4, rscratch1" %}
9481
9482 ins_encode %{
9483 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9484 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9485
9486 ins_pipe(imac_reg_reg);
9487 %}
9488
9489 // Integer Divide
9490
9491 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9492 match(Set dst (DivI src1 src2));
9493
9494 ins_cost(INSN_COST * 19);
9495 format %{ "sdivw $dst, $src1, $src2" %}
9496
9497 ins_encode(aarch64_enc_divw(dst, src1, src2));
9498 ins_pipe(idiv_reg_reg);
9499 %}
9500
9501 // Long Divide
9502
9503 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9504 match(Set dst (DivL src1 src2));
9505
9506 ins_cost(INSN_COST * 35);
9507 format %{ "sdiv $dst, $src1, $src2" %}
9508
9509 ins_encode(aarch64_enc_div(dst, src1, src2));
9510 ins_pipe(ldiv_reg_reg);
9511 %}
9512
9513 // Integer Remainder
9514
9515 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9516 match(Set dst (ModI src1 src2));
9517
9518 ins_cost(INSN_COST * 22);
9519 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9520 "msubw $dst, rscratch1, $src2, $src1" %}
9521
9522 ins_encode(aarch64_enc_modw(dst, src1, src2));
9523 ins_pipe(idiv_reg_reg);
9524 %}
9525
9526 // Long Remainder
9527
9528 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9529 match(Set dst (ModL src1 src2));
9530
9531 ins_cost(INSN_COST * 38);
9532 format %{ "sdiv rscratch1, $src1, $src2\n"
9533 "msub $dst, rscratch1, $src2, $src1" %}
9534
9535 ins_encode(aarch64_enc_mod(dst, src1, src2));
9536 ins_pipe(ldiv_reg_reg);
9537 %}
9538
9539 // Unsigned Integer Divide
9540
9541 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9542 match(Set dst (UDivI src1 src2));
9543
9544 ins_cost(INSN_COST * 19);
9545 format %{ "udivw $dst, $src1, $src2" %}
9546
9547 ins_encode %{
9548 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9549 %}
9550
9551 ins_pipe(idiv_reg_reg);
9552 %}
9553
9554 // Unsigned Long Divide
9555
9556 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9557 match(Set dst (UDivL src1 src2));
9558
9559 ins_cost(INSN_COST * 35);
9560 format %{ "udiv $dst, $src1, $src2" %}
9561
9562 ins_encode %{
9563 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9564 %}
9565
9566 ins_pipe(ldiv_reg_reg);
9567 %}
9568
9569 // Unsigned Integer Remainder
9570
9571 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9572 match(Set dst (UModI src1 src2));
9573
9574 ins_cost(INSN_COST * 22);
9575 format %{ "udivw rscratch1, $src1, $src2\n\t"
9576 "msubw $dst, rscratch1, $src2, $src1" %}
9577
9578 ins_encode %{
9579 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9580 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9581 %}
9582
9583 ins_pipe(idiv_reg_reg);
9584 %}
9585
9586 // Unsigned Long Remainder
9587
9588 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9589 match(Set dst (UModL src1 src2));
9590
9591 ins_cost(INSN_COST * 38);
9592 format %{ "udiv rscratch1, $src1, $src2\n"
9593 "msub $dst, rscratch1, $src2, $src1" %}
9594
9595 ins_encode %{
9596 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9597 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9598 %}
9599
9600 ins_pipe(ldiv_reg_reg);
9601 %}
9602
9603 // Integer Shifts
9604
9605 // Shift Left Register
9606 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9607 match(Set dst (LShiftI src1 src2));
9608
9609 ins_cost(INSN_COST * 2);
9610 format %{ "lslvw $dst, $src1, $src2" %}
9611
9612 ins_encode %{
9613 __ lslvw(as_Register($dst$$reg),
9614 as_Register($src1$$reg),
9615 as_Register($src2$$reg));
9616 %}
9617
9618 ins_pipe(ialu_reg_reg_vshift);
9619 %}
9620
9621 // Shift Left Immediate
9622 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9623 match(Set dst (LShiftI src1 src2));
9624
9625 ins_cost(INSN_COST);
9626 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9627
9628 ins_encode %{
9629 __ lslw(as_Register($dst$$reg),
9630 as_Register($src1$$reg),
9631 $src2$$constant & 0x1f);
9632 %}
9633
9634 ins_pipe(ialu_reg_shift);
9635 %}
9636
9637 // Shift Right Logical Register
9638 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9639 match(Set dst (URShiftI src1 src2));
9640
9641 ins_cost(INSN_COST * 2);
9642 format %{ "lsrvw $dst, $src1, $src2" %}
9643
9644 ins_encode %{
9645 __ lsrvw(as_Register($dst$$reg),
9646 as_Register($src1$$reg),
9647 as_Register($src2$$reg));
9648 %}
9649
9650 ins_pipe(ialu_reg_reg_vshift);
9651 %}
9652
9653 // Shift Right Logical Immediate
9654 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9655 match(Set dst (URShiftI src1 src2));
9656
9657 ins_cost(INSN_COST);
9658 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9659
9660 ins_encode %{
9661 __ lsrw(as_Register($dst$$reg),
9662 as_Register($src1$$reg),
9663 $src2$$constant & 0x1f);
9664 %}
9665
9666 ins_pipe(ialu_reg_shift);
9667 %}
9668
9669 // Shift Right Arithmetic Register
9670 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9671 match(Set dst (RShiftI src1 src2));
9672
9673 ins_cost(INSN_COST * 2);
9674 format %{ "asrvw $dst, $src1, $src2" %}
9675
9676 ins_encode %{
9677 __ asrvw(as_Register($dst$$reg),
9678 as_Register($src1$$reg),
9679 as_Register($src2$$reg));
9680 %}
9681
9682 ins_pipe(ialu_reg_reg_vshift);
9683 %}
9684
9685 // Shift Right Arithmetic Immediate
9686 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9687 match(Set dst (RShiftI src1 src2));
9688
9689 ins_cost(INSN_COST);
9690 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9691
9692 ins_encode %{
9693 __ asrw(as_Register($dst$$reg),
9694 as_Register($src1$$reg),
9695 $src2$$constant & 0x1f);
9696 %}
9697
9698 ins_pipe(ialu_reg_shift);
9699 %}
9700
9701 // Combined Int Mask and Right Shift (using UBFM)
9702 // TODO
9703
9704 // Long Shifts
9705
9706 // Shift Left Register
9707 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9708 match(Set dst (LShiftL src1 src2));
9709
9710 ins_cost(INSN_COST * 2);
9711 format %{ "lslv $dst, $src1, $src2" %}
9712
9713 ins_encode %{
9714 __ lslv(as_Register($dst$$reg),
9715 as_Register($src1$$reg),
9716 as_Register($src2$$reg));
9717 %}
9718
9719 ins_pipe(ialu_reg_reg_vshift);
9720 %}
9721
9722 // Shift Left Immediate
9723 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9724 match(Set dst (LShiftL src1 src2));
9725
9726 ins_cost(INSN_COST);
9727 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9728
9729 ins_encode %{
9730 __ lsl(as_Register($dst$$reg),
9731 as_Register($src1$$reg),
9732 $src2$$constant & 0x3f);
9733 %}
9734
9735 ins_pipe(ialu_reg_shift);
9736 %}
9737
9738 // Shift Right Logical Register
9739 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9740 match(Set dst (URShiftL src1 src2));
9741
9742 ins_cost(INSN_COST * 2);
9743 format %{ "lsrv $dst, $src1, $src2" %}
9744
9745 ins_encode %{
9746 __ lsrv(as_Register($dst$$reg),
9747 as_Register($src1$$reg),
9748 as_Register($src2$$reg));
9749 %}
9750
9751 ins_pipe(ialu_reg_reg_vshift);
9752 %}
9753
9754 // Shift Right Logical Immediate
9755 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9756 match(Set dst (URShiftL src1 src2));
9757
9758 ins_cost(INSN_COST);
9759 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9760
9761 ins_encode %{
9762 __ lsr(as_Register($dst$$reg),
9763 as_Register($src1$$reg),
9764 $src2$$constant & 0x3f);
9765 %}
9766
9767 ins_pipe(ialu_reg_shift);
9768 %}
9769
9770 // A special-case pattern for card table stores.
9771 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9772 match(Set dst (URShiftL (CastP2X src1) src2));
9773
9774 ins_cost(INSN_COST);
9775 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9776
9777 ins_encode %{
9778 __ lsr(as_Register($dst$$reg),
9779 as_Register($src1$$reg),
9780 $src2$$constant & 0x3f);
9781 %}
9782
9783 ins_pipe(ialu_reg_shift);
9784 %}
9785
9786 // Shift Right Arithmetic Register
9787 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9788 match(Set dst (RShiftL src1 src2));
9789
9790 ins_cost(INSN_COST * 2);
9791 format %{ "asrv $dst, $src1, $src2" %}
9792
9793 ins_encode %{
9794 __ asrv(as_Register($dst$$reg),
9795 as_Register($src1$$reg),
9796 as_Register($src2$$reg));
9797 %}
9798
9799 ins_pipe(ialu_reg_reg_vshift);
9800 %}
9801
9802 // Shift Right Arithmetic Immediate
9803 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9804 match(Set dst (RShiftL src1 src2));
9805
9806 ins_cost(INSN_COST);
9807 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9808
9809 ins_encode %{
9810 __ asr(as_Register($dst$$reg),
9811 as_Register($src1$$reg),
9812 $src2$$constant & 0x3f);
9813 %}
9814
9815 ins_pipe(ialu_reg_shift);
9816 %}
9817
9818 // BEGIN This section of the file is automatically generated. Do not edit --------------
9819 // This section is generated from aarch64_ad.m4
9820
9821 // This pattern is automatically generated from aarch64_ad.m4
9822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9823 instruct regL_not_reg(iRegLNoSp dst,
9824 iRegL src1, immL_M1 m1,
9825 rFlagsReg cr) %{
9826 match(Set dst (XorL src1 m1));
9827 ins_cost(INSN_COST);
9828 format %{ "eon $dst, $src1, zr" %}
9829
9830 ins_encode %{
9831 __ eon(as_Register($dst$$reg),
9832 as_Register($src1$$reg),
9833 zr,
9834 Assembler::LSL, 0);
9835 %}
9836
9837 ins_pipe(ialu_reg);
9838 %}
9839
9840 // This pattern is automatically generated from aarch64_ad.m4
9841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9842 instruct regI_not_reg(iRegINoSp dst,
9843 iRegIorL2I src1, immI_M1 m1,
9844 rFlagsReg cr) %{
9845 match(Set dst (XorI src1 m1));
9846 ins_cost(INSN_COST);
9847 format %{ "eonw $dst, $src1, zr" %}
9848
9849 ins_encode %{
9850 __ eonw(as_Register($dst$$reg),
9851 as_Register($src1$$reg),
9852 zr,
9853 Assembler::LSL, 0);
9854 %}
9855
9856 ins_pipe(ialu_reg);
9857 %}
9858
9859 // This pattern is automatically generated from aarch64_ad.m4
9860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9861 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9862 immI0 zero, iRegIorL2I src1, immI src2) %{
9863 match(Set dst (SubI zero (URShiftI src1 src2)));
9864
9865 ins_cost(1.9 * INSN_COST);
9866 format %{ "negw $dst, $src1, LSR $src2" %}
9867
9868 ins_encode %{
9869 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9870 Assembler::LSR, $src2$$constant & 0x1f);
9871 %}
9872
9873 ins_pipe(ialu_reg_shift);
9874 %}
9875
9876 // This pattern is automatically generated from aarch64_ad.m4
9877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9878 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9879 immI0 zero, iRegIorL2I src1, immI src2) %{
9880 match(Set dst (SubI zero (RShiftI src1 src2)));
9881
9882 ins_cost(1.9 * INSN_COST);
9883 format %{ "negw $dst, $src1, ASR $src2" %}
9884
9885 ins_encode %{
9886 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9887 Assembler::ASR, $src2$$constant & 0x1f);
9888 %}
9889
9890 ins_pipe(ialu_reg_shift);
9891 %}
9892
9893 // This pattern is automatically generated from aarch64_ad.m4
9894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9895 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9896 immI0 zero, iRegIorL2I src1, immI src2) %{
9897 match(Set dst (SubI zero (LShiftI src1 src2)));
9898
9899 ins_cost(1.9 * INSN_COST);
9900 format %{ "negw $dst, $src1, LSL $src2" %}
9901
9902 ins_encode %{
9903 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9904 Assembler::LSL, $src2$$constant & 0x1f);
9905 %}
9906
9907 ins_pipe(ialu_reg_shift);
9908 %}
9909
9910 // This pattern is automatically generated from aarch64_ad.m4
9911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9912 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9913 immL0 zero, iRegL src1, immI src2) %{
9914 match(Set dst (SubL zero (URShiftL src1 src2)));
9915
9916 ins_cost(1.9 * INSN_COST);
9917 format %{ "neg $dst, $src1, LSR $src2" %}
9918
9919 ins_encode %{
9920 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9921 Assembler::LSR, $src2$$constant & 0x3f);
9922 %}
9923
9924 ins_pipe(ialu_reg_shift);
9925 %}
9926
9927 // This pattern is automatically generated from aarch64_ad.m4
9928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9929 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9930 immL0 zero, iRegL src1, immI src2) %{
9931 match(Set dst (SubL zero (RShiftL src1 src2)));
9932
9933 ins_cost(1.9 * INSN_COST);
9934 format %{ "neg $dst, $src1, ASR $src2" %}
9935
9936 ins_encode %{
9937 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9938 Assembler::ASR, $src2$$constant & 0x3f);
9939 %}
9940
9941 ins_pipe(ialu_reg_shift);
9942 %}
9943
9944 // This pattern is automatically generated from aarch64_ad.m4
9945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9946 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
9947 immL0 zero, iRegL src1, immI src2) %{
9948 match(Set dst (SubL zero (LShiftL src1 src2)));
9949
9950 ins_cost(1.9 * INSN_COST);
9951 format %{ "neg $dst, $src1, LSL $src2" %}
9952
9953 ins_encode %{
9954 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9955 Assembler::LSL, $src2$$constant & 0x3f);
9956 %}
9957
9958 ins_pipe(ialu_reg_shift);
9959 %}
9960
9961 // This pattern is automatically generated from aarch64_ad.m4
9962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9963 instruct AndI_reg_not_reg(iRegINoSp dst,
9964 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
9965 match(Set dst (AndI src1 (XorI src2 m1)));
9966 ins_cost(INSN_COST);
9967 format %{ "bicw $dst, $src1, $src2" %}
9968
9969 ins_encode %{
9970 __ bicw(as_Register($dst$$reg),
9971 as_Register($src1$$reg),
9972 as_Register($src2$$reg),
9973 Assembler::LSL, 0);
9974 %}
9975
9976 ins_pipe(ialu_reg_reg);
9977 %}
9978
9979 // This pattern is automatically generated from aarch64_ad.m4
9980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9981 instruct AndL_reg_not_reg(iRegLNoSp dst,
9982 iRegL src1, iRegL src2, immL_M1 m1) %{
9983 match(Set dst (AndL src1 (XorL src2 m1)));
9984 ins_cost(INSN_COST);
9985 format %{ "bic $dst, $src1, $src2" %}
9986
9987 ins_encode %{
9988 __ bic(as_Register($dst$$reg),
9989 as_Register($src1$$reg),
9990 as_Register($src2$$reg),
9991 Assembler::LSL, 0);
9992 %}
9993
9994 ins_pipe(ialu_reg_reg);
9995 %}
9996
9997 // This pattern is automatically generated from aarch64_ad.m4
9998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9999 instruct OrI_reg_not_reg(iRegINoSp dst,
10000 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10001 match(Set dst (OrI src1 (XorI src2 m1)));
10002 ins_cost(INSN_COST);
10003 format %{ "ornw $dst, $src1, $src2" %}
10004
10005 ins_encode %{
10006 __ ornw(as_Register($dst$$reg),
10007 as_Register($src1$$reg),
10008 as_Register($src2$$reg),
10009 Assembler::LSL, 0);
10010 %}
10011
10012 ins_pipe(ialu_reg_reg);
10013 %}
10014
10015 // This pattern is automatically generated from aarch64_ad.m4
10016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10017 instruct OrL_reg_not_reg(iRegLNoSp dst,
10018 iRegL src1, iRegL src2, immL_M1 m1) %{
10019 match(Set dst (OrL src1 (XorL src2 m1)));
10020 ins_cost(INSN_COST);
10021 format %{ "orn $dst, $src1, $src2" %}
10022
10023 ins_encode %{
10024 __ orn(as_Register($dst$$reg),
10025 as_Register($src1$$reg),
10026 as_Register($src2$$reg),
10027 Assembler::LSL, 0);
10028 %}
10029
10030 ins_pipe(ialu_reg_reg);
10031 %}
10032
10033 // This pattern is automatically generated from aarch64_ad.m4
10034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10035 instruct XorI_reg_not_reg(iRegINoSp dst,
10036 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10037 match(Set dst (XorI m1 (XorI src2 src1)));
10038 ins_cost(INSN_COST);
10039 format %{ "eonw $dst, $src1, $src2" %}
10040
10041 ins_encode %{
10042 __ eonw(as_Register($dst$$reg),
10043 as_Register($src1$$reg),
10044 as_Register($src2$$reg),
10045 Assembler::LSL, 0);
10046 %}
10047
10048 ins_pipe(ialu_reg_reg);
10049 %}
10050
10051 // This pattern is automatically generated from aarch64_ad.m4
10052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10053 instruct XorL_reg_not_reg(iRegLNoSp dst,
10054 iRegL src1, iRegL src2, immL_M1 m1) %{
10055 match(Set dst (XorL m1 (XorL src2 src1)));
10056 ins_cost(INSN_COST);
10057 format %{ "eon $dst, $src1, $src2" %}
10058
10059 ins_encode %{
10060 __ eon(as_Register($dst$$reg),
10061 as_Register($src1$$reg),
10062 as_Register($src2$$reg),
10063 Assembler::LSL, 0);
10064 %}
10065
10066 ins_pipe(ialu_reg_reg);
10067 %}
10068
10069 // This pattern is automatically generated from aarch64_ad.m4
10070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10071 // val & (-1 ^ (val >>> shift)) ==> bicw
10072 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10073 iRegIorL2I src1, iRegIorL2I src2,
10074 immI src3, immI_M1 src4) %{
10075 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10076 ins_cost(1.9 * INSN_COST);
10077 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10078
10079 ins_encode %{
10080 __ bicw(as_Register($dst$$reg),
10081 as_Register($src1$$reg),
10082 as_Register($src2$$reg),
10083 Assembler::LSR,
10084 $src3$$constant & 0x1f);
10085 %}
10086
10087 ins_pipe(ialu_reg_reg_shift);
10088 %}
10089
10090 // This pattern is automatically generated from aarch64_ad.m4
10091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10092 // val & (-1 ^ (val >>> shift)) ==> bic
10093 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10094 iRegL src1, iRegL src2,
10095 immI src3, immL_M1 src4) %{
10096 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10097 ins_cost(1.9 * INSN_COST);
10098 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10099
10100 ins_encode %{
10101 __ bic(as_Register($dst$$reg),
10102 as_Register($src1$$reg),
10103 as_Register($src2$$reg),
10104 Assembler::LSR,
10105 $src3$$constant & 0x3f);
10106 %}
10107
10108 ins_pipe(ialu_reg_reg_shift);
10109 %}
10110
10111 // This pattern is automatically generated from aarch64_ad.m4
10112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10113 // val & (-1 ^ (val >> shift)) ==> bicw
10114 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10115 iRegIorL2I src1, iRegIorL2I src2,
10116 immI src3, immI_M1 src4) %{
10117 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10118 ins_cost(1.9 * INSN_COST);
10119 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10120
10121 ins_encode %{
10122 __ bicw(as_Register($dst$$reg),
10123 as_Register($src1$$reg),
10124 as_Register($src2$$reg),
10125 Assembler::ASR,
10126 $src3$$constant & 0x1f);
10127 %}
10128
10129 ins_pipe(ialu_reg_reg_shift);
10130 %}
10131
10132 // This pattern is automatically generated from aarch64_ad.m4
10133 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10134 // val & (-1 ^ (val >> shift)) ==> bic
10135 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10136 iRegL src1, iRegL src2,
10137 immI src3, immL_M1 src4) %{
10138 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10139 ins_cost(1.9 * INSN_COST);
10140 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10141
10142 ins_encode %{
10143 __ bic(as_Register($dst$$reg),
10144 as_Register($src1$$reg),
10145 as_Register($src2$$reg),
10146 Assembler::ASR,
10147 $src3$$constant & 0x3f);
10148 %}
10149
10150 ins_pipe(ialu_reg_reg_shift);
10151 %}
10152
10153 // This pattern is automatically generated from aarch64_ad.m4
10154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10155 // val & (-1 ^ (val ror shift)) ==> bicw
10156 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10157 iRegIorL2I src1, iRegIorL2I src2,
10158 immI src3, immI_M1 src4) %{
10159 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10160 ins_cost(1.9 * INSN_COST);
10161 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10162
10163 ins_encode %{
10164 __ bicw(as_Register($dst$$reg),
10165 as_Register($src1$$reg),
10166 as_Register($src2$$reg),
10167 Assembler::ROR,
10168 $src3$$constant & 0x1f);
10169 %}
10170
10171 ins_pipe(ialu_reg_reg_shift);
10172 %}
10173
10174 // This pattern is automatically generated from aarch64_ad.m4
10175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10176 // val & (-1 ^ (val ror shift)) ==> bic
10177 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10178 iRegL src1, iRegL src2,
10179 immI src3, immL_M1 src4) %{
10180 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10181 ins_cost(1.9 * INSN_COST);
10182 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10183
10184 ins_encode %{
10185 __ bic(as_Register($dst$$reg),
10186 as_Register($src1$$reg),
10187 as_Register($src2$$reg),
10188 Assembler::ROR,
10189 $src3$$constant & 0x3f);
10190 %}
10191
10192 ins_pipe(ialu_reg_reg_shift);
10193 %}
10194
10195 // This pattern is automatically generated from aarch64_ad.m4
10196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10197 // val & (-1 ^ (val << shift)) ==> bicw
10198 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10199 iRegIorL2I src1, iRegIorL2I src2,
10200 immI src3, immI_M1 src4) %{
10201 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10202 ins_cost(1.9 * INSN_COST);
10203 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10204
10205 ins_encode %{
10206 __ bicw(as_Register($dst$$reg),
10207 as_Register($src1$$reg),
10208 as_Register($src2$$reg),
10209 Assembler::LSL,
10210 $src3$$constant & 0x1f);
10211 %}
10212
10213 ins_pipe(ialu_reg_reg_shift);
10214 %}
10215
10216 // This pattern is automatically generated from aarch64_ad.m4
10217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10218 // val & (-1 ^ (val << shift)) ==> bic
10219 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10220 iRegL src1, iRegL src2,
10221 immI src3, immL_M1 src4) %{
10222 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10223 ins_cost(1.9 * INSN_COST);
10224 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10225
10226 ins_encode %{
10227 __ bic(as_Register($dst$$reg),
10228 as_Register($src1$$reg),
10229 as_Register($src2$$reg),
10230 Assembler::LSL,
10231 $src3$$constant & 0x3f);
10232 %}
10233
10234 ins_pipe(ialu_reg_reg_shift);
10235 %}
10236
10237 // This pattern is automatically generated from aarch64_ad.m4
10238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10239 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10240 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10241 iRegIorL2I src1, iRegIorL2I src2,
10242 immI src3, immI_M1 src4) %{
10243 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10244 ins_cost(1.9 * INSN_COST);
10245 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10246
10247 ins_encode %{
10248 __ eonw(as_Register($dst$$reg),
10249 as_Register($src1$$reg),
10250 as_Register($src2$$reg),
10251 Assembler::LSR,
10252 $src3$$constant & 0x1f);
10253 %}
10254
10255 ins_pipe(ialu_reg_reg_shift);
10256 %}
10257
10258 // This pattern is automatically generated from aarch64_ad.m4
10259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10260 // val ^ (-1 ^ (val >>> shift)) ==> eon
10261 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10262 iRegL src1, iRegL src2,
10263 immI src3, immL_M1 src4) %{
10264 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10265 ins_cost(1.9 * INSN_COST);
10266 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10267
10268 ins_encode %{
10269 __ eon(as_Register($dst$$reg),
10270 as_Register($src1$$reg),
10271 as_Register($src2$$reg),
10272 Assembler::LSR,
10273 $src3$$constant & 0x3f);
10274 %}
10275
10276 ins_pipe(ialu_reg_reg_shift);
10277 %}
10278
10279 // This pattern is automatically generated from aarch64_ad.m4
10280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10281 // val ^ (-1 ^ (val >> shift)) ==> eonw
10282 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10283 iRegIorL2I src1, iRegIorL2I src2,
10284 immI src3, immI_M1 src4) %{
10285 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10286 ins_cost(1.9 * INSN_COST);
10287 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10288
10289 ins_encode %{
10290 __ eonw(as_Register($dst$$reg),
10291 as_Register($src1$$reg),
10292 as_Register($src2$$reg),
10293 Assembler::ASR,
10294 $src3$$constant & 0x1f);
10295 %}
10296
10297 ins_pipe(ialu_reg_reg_shift);
10298 %}
10299
10300 // This pattern is automatically generated from aarch64_ad.m4
10301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10302 // val ^ (-1 ^ (val >> shift)) ==> eon
10303 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10304 iRegL src1, iRegL src2,
10305 immI src3, immL_M1 src4) %{
10306 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10307 ins_cost(1.9 * INSN_COST);
10308 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10309
10310 ins_encode %{
10311 __ eon(as_Register($dst$$reg),
10312 as_Register($src1$$reg),
10313 as_Register($src2$$reg),
10314 Assembler::ASR,
10315 $src3$$constant & 0x3f);
10316 %}
10317
10318 ins_pipe(ialu_reg_reg_shift);
10319 %}
10320
10321 // This pattern is automatically generated from aarch64_ad.m4
10322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10323 // val ^ (-1 ^ (val ror shift)) ==> eonw
10324 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10325 iRegIorL2I src1, iRegIorL2I src2,
10326 immI src3, immI_M1 src4) %{
10327 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10328 ins_cost(1.9 * INSN_COST);
10329 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10330
10331 ins_encode %{
10332 __ eonw(as_Register($dst$$reg),
10333 as_Register($src1$$reg),
10334 as_Register($src2$$reg),
10335 Assembler::ROR,
10336 $src3$$constant & 0x1f);
10337 %}
10338
10339 ins_pipe(ialu_reg_reg_shift);
10340 %}
10341
10342 // This pattern is automatically generated from aarch64_ad.m4
10343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10344 // val ^ (-1 ^ (val ror shift)) ==> eon
10345 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10346 iRegL src1, iRegL src2,
10347 immI src3, immL_M1 src4) %{
10348 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10349 ins_cost(1.9 * INSN_COST);
10350 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10351
10352 ins_encode %{
10353 __ eon(as_Register($dst$$reg),
10354 as_Register($src1$$reg),
10355 as_Register($src2$$reg),
10356 Assembler::ROR,
10357 $src3$$constant & 0x3f);
10358 %}
10359
10360 ins_pipe(ialu_reg_reg_shift);
10361 %}
10362
10363 // This pattern is automatically generated from aarch64_ad.m4
10364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10365 // val ^ (-1 ^ (val << shift)) ==> eonw
10366 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10367 iRegIorL2I src1, iRegIorL2I src2,
10368 immI src3, immI_M1 src4) %{
10369 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10370 ins_cost(1.9 * INSN_COST);
10371 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10372
10373 ins_encode %{
10374 __ eonw(as_Register($dst$$reg),
10375 as_Register($src1$$reg),
10376 as_Register($src2$$reg),
10377 Assembler::LSL,
10378 $src3$$constant & 0x1f);
10379 %}
10380
10381 ins_pipe(ialu_reg_reg_shift);
10382 %}
10383
10384 // This pattern is automatically generated from aarch64_ad.m4
10385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10386 // val ^ (-1 ^ (val << shift)) ==> eon
10387 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10388 iRegL src1, iRegL src2,
10389 immI src3, immL_M1 src4) %{
10390 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10391 ins_cost(1.9 * INSN_COST);
10392 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10393
10394 ins_encode %{
10395 __ eon(as_Register($dst$$reg),
10396 as_Register($src1$$reg),
10397 as_Register($src2$$reg),
10398 Assembler::LSL,
10399 $src3$$constant & 0x3f);
10400 %}
10401
10402 ins_pipe(ialu_reg_reg_shift);
10403 %}
10404
10405 // This pattern is automatically generated from aarch64_ad.m4
10406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10407 // val | (-1 ^ (val >>> shift)) ==> ornw
10408 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10409 iRegIorL2I src1, iRegIorL2I src2,
10410 immI src3, immI_M1 src4) %{
10411 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10412 ins_cost(1.9 * INSN_COST);
10413 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10414
10415 ins_encode %{
10416 __ ornw(as_Register($dst$$reg),
10417 as_Register($src1$$reg),
10418 as_Register($src2$$reg),
10419 Assembler::LSR,
10420 $src3$$constant & 0x1f);
10421 %}
10422
10423 ins_pipe(ialu_reg_reg_shift);
10424 %}
10425
10426 // This pattern is automatically generated from aarch64_ad.m4
10427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10428 // val | (-1 ^ (val >>> shift)) ==> orn
10429 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10430 iRegL src1, iRegL src2,
10431 immI src3, immL_M1 src4) %{
10432 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10433 ins_cost(1.9 * INSN_COST);
10434 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10435
10436 ins_encode %{
10437 __ orn(as_Register($dst$$reg),
10438 as_Register($src1$$reg),
10439 as_Register($src2$$reg),
10440 Assembler::LSR,
10441 $src3$$constant & 0x3f);
10442 %}
10443
10444 ins_pipe(ialu_reg_reg_shift);
10445 %}
10446
10447 // This pattern is automatically generated from aarch64_ad.m4
10448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10449 // val | (-1 ^ (val >> shift)) ==> ornw
10450 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10451 iRegIorL2I src1, iRegIorL2I src2,
10452 immI src3, immI_M1 src4) %{
10453 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10454 ins_cost(1.9 * INSN_COST);
10455 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10456
10457 ins_encode %{
10458 __ ornw(as_Register($dst$$reg),
10459 as_Register($src1$$reg),
10460 as_Register($src2$$reg),
10461 Assembler::ASR,
10462 $src3$$constant & 0x1f);
10463 %}
10464
10465 ins_pipe(ialu_reg_reg_shift);
10466 %}
10467
10468 // This pattern is automatically generated from aarch64_ad.m4
10469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10470 // val | (-1 ^ (val >> shift)) ==> orn
10471 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10472 iRegL src1, iRegL src2,
10473 immI src3, immL_M1 src4) %{
10474 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10475 ins_cost(1.9 * INSN_COST);
10476 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10477
10478 ins_encode %{
10479 __ orn(as_Register($dst$$reg),
10480 as_Register($src1$$reg),
10481 as_Register($src2$$reg),
10482 Assembler::ASR,
10483 $src3$$constant & 0x3f);
10484 %}
10485
10486 ins_pipe(ialu_reg_reg_shift);
10487 %}
10488
10489 // This pattern is automatically generated from aarch64_ad.m4
10490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10491 // val | (-1 ^ (val ror shift)) ==> ornw
10492 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10493 iRegIorL2I src1, iRegIorL2I src2,
10494 immI src3, immI_M1 src4) %{
10495 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10496 ins_cost(1.9 * INSN_COST);
10497 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10498
10499 ins_encode %{
10500 __ ornw(as_Register($dst$$reg),
10501 as_Register($src1$$reg),
10502 as_Register($src2$$reg),
10503 Assembler::ROR,
10504 $src3$$constant & 0x1f);
10505 %}
10506
10507 ins_pipe(ialu_reg_reg_shift);
10508 %}
10509
10510 // This pattern is automatically generated from aarch64_ad.m4
10511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10512 // val | (-1 ^ (val ror shift)) ==> orn
10513 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10514 iRegL src1, iRegL src2,
10515 immI src3, immL_M1 src4) %{
10516 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10517 ins_cost(1.9 * INSN_COST);
10518 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10519
10520 ins_encode %{
10521 __ orn(as_Register($dst$$reg),
10522 as_Register($src1$$reg),
10523 as_Register($src2$$reg),
10524 Assembler::ROR,
10525 $src3$$constant & 0x3f);
10526 %}
10527
10528 ins_pipe(ialu_reg_reg_shift);
10529 %}
10530
10531 // This pattern is automatically generated from aarch64_ad.m4
10532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10533 // val | (-1 ^ (val << shift)) ==> ornw
10534 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10535 iRegIorL2I src1, iRegIorL2I src2,
10536 immI src3, immI_M1 src4) %{
10537 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10538 ins_cost(1.9 * INSN_COST);
10539 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10540
10541 ins_encode %{
10542 __ ornw(as_Register($dst$$reg),
10543 as_Register($src1$$reg),
10544 as_Register($src2$$reg),
10545 Assembler::LSL,
10546 $src3$$constant & 0x1f);
10547 %}
10548
10549 ins_pipe(ialu_reg_reg_shift);
10550 %}
10551
10552 // This pattern is automatically generated from aarch64_ad.m4
10553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10554 // val | (-1 ^ (val << shift)) ==> orn
10555 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10556 iRegL src1, iRegL src2,
10557 immI src3, immL_M1 src4) %{
10558 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10559 ins_cost(1.9 * INSN_COST);
10560 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10561
10562 ins_encode %{
10563 __ orn(as_Register($dst$$reg),
10564 as_Register($src1$$reg),
10565 as_Register($src2$$reg),
10566 Assembler::LSL,
10567 $src3$$constant & 0x3f);
10568 %}
10569
10570 ins_pipe(ialu_reg_reg_shift);
10571 %}
10572
10573 // This pattern is automatically generated from aarch64_ad.m4
10574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10575 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10576 iRegIorL2I src1, iRegIorL2I src2,
10577 immI src3) %{
10578 match(Set dst (AndI src1 (URShiftI src2 src3)));
10579
10580 ins_cost(1.9 * INSN_COST);
10581 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10582
10583 ins_encode %{
10584 __ andw(as_Register($dst$$reg),
10585 as_Register($src1$$reg),
10586 as_Register($src2$$reg),
10587 Assembler::LSR,
10588 $src3$$constant & 0x1f);
10589 %}
10590
10591 ins_pipe(ialu_reg_reg_shift);
10592 %}
10593
10594 // This pattern is automatically generated from aarch64_ad.m4
10595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10596 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10597 iRegL src1, iRegL src2,
10598 immI src3) %{
10599 match(Set dst (AndL src1 (URShiftL src2 src3)));
10600
10601 ins_cost(1.9 * INSN_COST);
10602 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10603
10604 ins_encode %{
10605 __ andr(as_Register($dst$$reg),
10606 as_Register($src1$$reg),
10607 as_Register($src2$$reg),
10608 Assembler::LSR,
10609 $src3$$constant & 0x3f);
10610 %}
10611
10612 ins_pipe(ialu_reg_reg_shift);
10613 %}
10614
10615 // This pattern is automatically generated from aarch64_ad.m4
10616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10617 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10618 iRegIorL2I src1, iRegIorL2I src2,
10619 immI src3) %{
10620 match(Set dst (AndI src1 (RShiftI src2 src3)));
10621
10622 ins_cost(1.9 * INSN_COST);
10623 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10624
10625 ins_encode %{
10626 __ andw(as_Register($dst$$reg),
10627 as_Register($src1$$reg),
10628 as_Register($src2$$reg),
10629 Assembler::ASR,
10630 $src3$$constant & 0x1f);
10631 %}
10632
10633 ins_pipe(ialu_reg_reg_shift);
10634 %}
10635
10636 // This pattern is automatically generated from aarch64_ad.m4
10637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10638 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10639 iRegL src1, iRegL src2,
10640 immI src3) %{
10641 match(Set dst (AndL src1 (RShiftL src2 src3)));
10642
10643 ins_cost(1.9 * INSN_COST);
10644 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10645
10646 ins_encode %{
10647 __ andr(as_Register($dst$$reg),
10648 as_Register($src1$$reg),
10649 as_Register($src2$$reg),
10650 Assembler::ASR,
10651 $src3$$constant & 0x3f);
10652 %}
10653
10654 ins_pipe(ialu_reg_reg_shift);
10655 %}
10656
10657 // This pattern is automatically generated from aarch64_ad.m4
10658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10659 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10660 iRegIorL2I src1, iRegIorL2I src2,
10661 immI src3) %{
10662 match(Set dst (AndI src1 (LShiftI src2 src3)));
10663
10664 ins_cost(1.9 * INSN_COST);
10665 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10666
10667 ins_encode %{
10668 __ andw(as_Register($dst$$reg),
10669 as_Register($src1$$reg),
10670 as_Register($src2$$reg),
10671 Assembler::LSL,
10672 $src3$$constant & 0x1f);
10673 %}
10674
10675 ins_pipe(ialu_reg_reg_shift);
10676 %}
10677
10678 // This pattern is automatically generated from aarch64_ad.m4
10679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10680 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10681 iRegL src1, iRegL src2,
10682 immI src3) %{
10683 match(Set dst (AndL src1 (LShiftL src2 src3)));
10684
10685 ins_cost(1.9 * INSN_COST);
10686 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10687
10688 ins_encode %{
10689 __ andr(as_Register($dst$$reg),
10690 as_Register($src1$$reg),
10691 as_Register($src2$$reg),
10692 Assembler::LSL,
10693 $src3$$constant & 0x3f);
10694 %}
10695
10696 ins_pipe(ialu_reg_reg_shift);
10697 %}
10698
10699 // This pattern is automatically generated from aarch64_ad.m4
10700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10701 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10702 iRegIorL2I src1, iRegIorL2I src2,
10703 immI src3) %{
10704 match(Set dst (AndI src1 (RotateRight src2 src3)));
10705
10706 ins_cost(1.9 * INSN_COST);
10707 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10708
10709 ins_encode %{
10710 __ andw(as_Register($dst$$reg),
10711 as_Register($src1$$reg),
10712 as_Register($src2$$reg),
10713 Assembler::ROR,
10714 $src3$$constant & 0x1f);
10715 %}
10716
10717 ins_pipe(ialu_reg_reg_shift);
10718 %}
10719
10720 // This pattern is automatically generated from aarch64_ad.m4
10721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10722 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10723 iRegL src1, iRegL src2,
10724 immI src3) %{
10725 match(Set dst (AndL src1 (RotateRight src2 src3)));
10726
10727 ins_cost(1.9 * INSN_COST);
10728 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10729
10730 ins_encode %{
10731 __ andr(as_Register($dst$$reg),
10732 as_Register($src1$$reg),
10733 as_Register($src2$$reg),
10734 Assembler::ROR,
10735 $src3$$constant & 0x3f);
10736 %}
10737
10738 ins_pipe(ialu_reg_reg_shift);
10739 %}
10740
10741 // This pattern is automatically generated from aarch64_ad.m4
10742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10743 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10744 iRegIorL2I src1, iRegIorL2I src2,
10745 immI src3) %{
10746 match(Set dst (XorI src1 (URShiftI src2 src3)));
10747
10748 ins_cost(1.9 * INSN_COST);
10749 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10750
10751 ins_encode %{
10752 __ eorw(as_Register($dst$$reg),
10753 as_Register($src1$$reg),
10754 as_Register($src2$$reg),
10755 Assembler::LSR,
10756 $src3$$constant & 0x1f);
10757 %}
10758
10759 ins_pipe(ialu_reg_reg_shift);
10760 %}
10761
10762 // This pattern is automatically generated from aarch64_ad.m4
10763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10764 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10765 iRegL src1, iRegL src2,
10766 immI src3) %{
10767 match(Set dst (XorL src1 (URShiftL src2 src3)));
10768
10769 ins_cost(1.9 * INSN_COST);
10770 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10771
10772 ins_encode %{
10773 __ eor(as_Register($dst$$reg),
10774 as_Register($src1$$reg),
10775 as_Register($src2$$reg),
10776 Assembler::LSR,
10777 $src3$$constant & 0x3f);
10778 %}
10779
10780 ins_pipe(ialu_reg_reg_shift);
10781 %}
10782
10783 // This pattern is automatically generated from aarch64_ad.m4
10784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10785 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10786 iRegIorL2I src1, iRegIorL2I src2,
10787 immI src3) %{
10788 match(Set dst (XorI src1 (RShiftI src2 src3)));
10789
10790 ins_cost(1.9 * INSN_COST);
10791 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10792
10793 ins_encode %{
10794 __ eorw(as_Register($dst$$reg),
10795 as_Register($src1$$reg),
10796 as_Register($src2$$reg),
10797 Assembler::ASR,
10798 $src3$$constant & 0x1f);
10799 %}
10800
10801 ins_pipe(ialu_reg_reg_shift);
10802 %}
10803
10804 // This pattern is automatically generated from aarch64_ad.m4
10805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10806 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10807 iRegL src1, iRegL src2,
10808 immI src3) %{
10809 match(Set dst (XorL src1 (RShiftL src2 src3)));
10810
10811 ins_cost(1.9 * INSN_COST);
10812 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10813
10814 ins_encode %{
10815 __ eor(as_Register($dst$$reg),
10816 as_Register($src1$$reg),
10817 as_Register($src2$$reg),
10818 Assembler::ASR,
10819 $src3$$constant & 0x3f);
10820 %}
10821
10822 ins_pipe(ialu_reg_reg_shift);
10823 %}
10824
10825 // This pattern is automatically generated from aarch64_ad.m4
10826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10827 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10828 iRegIorL2I src1, iRegIorL2I src2,
10829 immI src3) %{
10830 match(Set dst (XorI src1 (LShiftI src2 src3)));
10831
10832 ins_cost(1.9 * INSN_COST);
10833 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10834
10835 ins_encode %{
10836 __ eorw(as_Register($dst$$reg),
10837 as_Register($src1$$reg),
10838 as_Register($src2$$reg),
10839 Assembler::LSL,
10840 $src3$$constant & 0x1f);
10841 %}
10842
10843 ins_pipe(ialu_reg_reg_shift);
10844 %}
10845
10846 // This pattern is automatically generated from aarch64_ad.m4
10847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10848 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10849 iRegL src1, iRegL src2,
10850 immI src3) %{
10851 match(Set dst (XorL src1 (LShiftL src2 src3)));
10852
10853 ins_cost(1.9 * INSN_COST);
10854 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10855
10856 ins_encode %{
10857 __ eor(as_Register($dst$$reg),
10858 as_Register($src1$$reg),
10859 as_Register($src2$$reg),
10860 Assembler::LSL,
10861 $src3$$constant & 0x3f);
10862 %}
10863
10864 ins_pipe(ialu_reg_reg_shift);
10865 %}
10866
10867 // This pattern is automatically generated from aarch64_ad.m4
10868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10869 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10870 iRegIorL2I src1, iRegIorL2I src2,
10871 immI src3) %{
10872 match(Set dst (XorI src1 (RotateRight src2 src3)));
10873
10874 ins_cost(1.9 * INSN_COST);
10875 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10876
10877 ins_encode %{
10878 __ eorw(as_Register($dst$$reg),
10879 as_Register($src1$$reg),
10880 as_Register($src2$$reg),
10881 Assembler::ROR,
10882 $src3$$constant & 0x1f);
10883 %}
10884
10885 ins_pipe(ialu_reg_reg_shift);
10886 %}
10887
10888 // This pattern is automatically generated from aarch64_ad.m4
10889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10890 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10891 iRegL src1, iRegL src2,
10892 immI src3) %{
10893 match(Set dst (XorL src1 (RotateRight src2 src3)));
10894
10895 ins_cost(1.9 * INSN_COST);
10896 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10897
10898 ins_encode %{
10899 __ eor(as_Register($dst$$reg),
10900 as_Register($src1$$reg),
10901 as_Register($src2$$reg),
10902 Assembler::ROR,
10903 $src3$$constant & 0x3f);
10904 %}
10905
10906 ins_pipe(ialu_reg_reg_shift);
10907 %}
10908
10909 // This pattern is automatically generated from aarch64_ad.m4
10910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10911 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10912 iRegIorL2I src1, iRegIorL2I src2,
10913 immI src3) %{
10914 match(Set dst (OrI src1 (URShiftI src2 src3)));
10915
10916 ins_cost(1.9 * INSN_COST);
10917 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10918
10919 ins_encode %{
10920 __ orrw(as_Register($dst$$reg),
10921 as_Register($src1$$reg),
10922 as_Register($src2$$reg),
10923 Assembler::LSR,
10924 $src3$$constant & 0x1f);
10925 %}
10926
10927 ins_pipe(ialu_reg_reg_shift);
10928 %}
10929
10930 // This pattern is automatically generated from aarch64_ad.m4
10931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10932 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10933 iRegL src1, iRegL src2,
10934 immI src3) %{
10935 match(Set dst (OrL src1 (URShiftL src2 src3)));
10936
10937 ins_cost(1.9 * INSN_COST);
10938 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10939
10940 ins_encode %{
10941 __ orr(as_Register($dst$$reg),
10942 as_Register($src1$$reg),
10943 as_Register($src2$$reg),
10944 Assembler::LSR,
10945 $src3$$constant & 0x3f);
10946 %}
10947
10948 ins_pipe(ialu_reg_reg_shift);
10949 %}
10950
10951 // This pattern is automatically generated from aarch64_ad.m4
10952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10953 instruct OrI_reg_RShift_reg(iRegINoSp dst,
10954 iRegIorL2I src1, iRegIorL2I src2,
10955 immI src3) %{
10956 match(Set dst (OrI src1 (RShiftI src2 src3)));
10957
10958 ins_cost(1.9 * INSN_COST);
10959 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
10960
10961 ins_encode %{
10962 __ orrw(as_Register($dst$$reg),
10963 as_Register($src1$$reg),
10964 as_Register($src2$$reg),
10965 Assembler::ASR,
10966 $src3$$constant & 0x1f);
10967 %}
10968
10969 ins_pipe(ialu_reg_reg_shift);
10970 %}
10971
10972 // This pattern is automatically generated from aarch64_ad.m4
10973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10974 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
10975 iRegL src1, iRegL src2,
10976 immI src3) %{
10977 match(Set dst (OrL src1 (RShiftL src2 src3)));
10978
10979 ins_cost(1.9 * INSN_COST);
10980 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
10981
10982 ins_encode %{
10983 __ orr(as_Register($dst$$reg),
10984 as_Register($src1$$reg),
10985 as_Register($src2$$reg),
10986 Assembler::ASR,
10987 $src3$$constant & 0x3f);
10988 %}
10989
10990 ins_pipe(ialu_reg_reg_shift);
10991 %}
10992
10993 // This pattern is automatically generated from aarch64_ad.m4
10994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10995 instruct OrI_reg_LShift_reg(iRegINoSp dst,
10996 iRegIorL2I src1, iRegIorL2I src2,
10997 immI src3) %{
10998 match(Set dst (OrI src1 (LShiftI src2 src3)));
10999
11000 ins_cost(1.9 * INSN_COST);
11001 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11002
11003 ins_encode %{
11004 __ orrw(as_Register($dst$$reg),
11005 as_Register($src1$$reg),
11006 as_Register($src2$$reg),
11007 Assembler::LSL,
11008 $src3$$constant & 0x1f);
11009 %}
11010
11011 ins_pipe(ialu_reg_reg_shift);
11012 %}
11013
11014 // This pattern is automatically generated from aarch64_ad.m4
11015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11016 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11017 iRegL src1, iRegL src2,
11018 immI src3) %{
11019 match(Set dst (OrL src1 (LShiftL src2 src3)));
11020
11021 ins_cost(1.9 * INSN_COST);
11022 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11023
11024 ins_encode %{
11025 __ orr(as_Register($dst$$reg),
11026 as_Register($src1$$reg),
11027 as_Register($src2$$reg),
11028 Assembler::LSL,
11029 $src3$$constant & 0x3f);
11030 %}
11031
11032 ins_pipe(ialu_reg_reg_shift);
11033 %}
11034
11035 // This pattern is automatically generated from aarch64_ad.m4
11036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11037 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11038 iRegIorL2I src1, iRegIorL2I src2,
11039 immI src3) %{
11040 match(Set dst (OrI src1 (RotateRight src2 src3)));
11041
11042 ins_cost(1.9 * INSN_COST);
11043 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11044
11045 ins_encode %{
11046 __ orrw(as_Register($dst$$reg),
11047 as_Register($src1$$reg),
11048 as_Register($src2$$reg),
11049 Assembler::ROR,
11050 $src3$$constant & 0x1f);
11051 %}
11052
11053 ins_pipe(ialu_reg_reg_shift);
11054 %}
11055
11056 // This pattern is automatically generated from aarch64_ad.m4
11057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11058 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11059 iRegL src1, iRegL src2,
11060 immI src3) %{
11061 match(Set dst (OrL src1 (RotateRight src2 src3)));
11062
11063 ins_cost(1.9 * INSN_COST);
11064 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11065
11066 ins_encode %{
11067 __ orr(as_Register($dst$$reg),
11068 as_Register($src1$$reg),
11069 as_Register($src2$$reg),
11070 Assembler::ROR,
11071 $src3$$constant & 0x3f);
11072 %}
11073
11074 ins_pipe(ialu_reg_reg_shift);
11075 %}
11076
11077 // This pattern is automatically generated from aarch64_ad.m4
11078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11079 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11080 iRegIorL2I src1, iRegIorL2I src2,
11081 immI src3) %{
11082 match(Set dst (AddI src1 (URShiftI src2 src3)));
11083
11084 ins_cost(1.9 * INSN_COST);
11085 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11086
11087 ins_encode %{
11088 __ addw(as_Register($dst$$reg),
11089 as_Register($src1$$reg),
11090 as_Register($src2$$reg),
11091 Assembler::LSR,
11092 $src3$$constant & 0x1f);
11093 %}
11094
11095 ins_pipe(ialu_reg_reg_shift);
11096 %}
11097
11098 // This pattern is automatically generated from aarch64_ad.m4
11099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11100 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11101 iRegL src1, iRegL src2,
11102 immI src3) %{
11103 match(Set dst (AddL src1 (URShiftL src2 src3)));
11104
11105 ins_cost(1.9 * INSN_COST);
11106 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11107
11108 ins_encode %{
11109 __ add(as_Register($dst$$reg),
11110 as_Register($src1$$reg),
11111 as_Register($src2$$reg),
11112 Assembler::LSR,
11113 $src3$$constant & 0x3f);
11114 %}
11115
11116 ins_pipe(ialu_reg_reg_shift);
11117 %}
11118
11119 // This pattern is automatically generated from aarch64_ad.m4
11120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11121 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11122 iRegIorL2I src1, iRegIorL2I src2,
11123 immI src3) %{
11124 match(Set dst (AddI src1 (RShiftI src2 src3)));
11125
11126 ins_cost(1.9 * INSN_COST);
11127 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11128
11129 ins_encode %{
11130 __ addw(as_Register($dst$$reg),
11131 as_Register($src1$$reg),
11132 as_Register($src2$$reg),
11133 Assembler::ASR,
11134 $src3$$constant & 0x1f);
11135 %}
11136
11137 ins_pipe(ialu_reg_reg_shift);
11138 %}
11139
11140 // This pattern is automatically generated from aarch64_ad.m4
11141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11142 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11143 iRegL src1, iRegL src2,
11144 immI src3) %{
11145 match(Set dst (AddL src1 (RShiftL src2 src3)));
11146
11147 ins_cost(1.9 * INSN_COST);
11148 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11149
11150 ins_encode %{
11151 __ add(as_Register($dst$$reg),
11152 as_Register($src1$$reg),
11153 as_Register($src2$$reg),
11154 Assembler::ASR,
11155 $src3$$constant & 0x3f);
11156 %}
11157
11158 ins_pipe(ialu_reg_reg_shift);
11159 %}
11160
11161 // This pattern is automatically generated from aarch64_ad.m4
11162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11163 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11164 iRegIorL2I src1, iRegIorL2I src2,
11165 immI src3) %{
11166 match(Set dst (AddI src1 (LShiftI src2 src3)));
11167
11168 ins_cost(1.9 * INSN_COST);
11169 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11170
11171 ins_encode %{
11172 __ addw(as_Register($dst$$reg),
11173 as_Register($src1$$reg),
11174 as_Register($src2$$reg),
11175 Assembler::LSL,
11176 $src3$$constant & 0x1f);
11177 %}
11178
11179 ins_pipe(ialu_reg_reg_shift);
11180 %}
11181
11182 // This pattern is automatically generated from aarch64_ad.m4
11183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11184 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11185 iRegL src1, iRegL src2,
11186 immI src3) %{
11187 match(Set dst (AddL src1 (LShiftL src2 src3)));
11188
11189 ins_cost(1.9 * INSN_COST);
11190 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11191
11192 ins_encode %{
11193 __ add(as_Register($dst$$reg),
11194 as_Register($src1$$reg),
11195 as_Register($src2$$reg),
11196 Assembler::LSL,
11197 $src3$$constant & 0x3f);
11198 %}
11199
11200 ins_pipe(ialu_reg_reg_shift);
11201 %}
11202
11203 // This pattern is automatically generated from aarch64_ad.m4
11204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11205 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11206 iRegIorL2I src1, iRegIorL2I src2,
11207 immI src3) %{
11208 match(Set dst (SubI src1 (URShiftI src2 src3)));
11209
11210 ins_cost(1.9 * INSN_COST);
11211 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11212
11213 ins_encode %{
11214 __ subw(as_Register($dst$$reg),
11215 as_Register($src1$$reg),
11216 as_Register($src2$$reg),
11217 Assembler::LSR,
11218 $src3$$constant & 0x1f);
11219 %}
11220
11221 ins_pipe(ialu_reg_reg_shift);
11222 %}
11223
11224 // This pattern is automatically generated from aarch64_ad.m4
11225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11226 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11227 iRegL src1, iRegL src2,
11228 immI src3) %{
11229 match(Set dst (SubL src1 (URShiftL src2 src3)));
11230
11231 ins_cost(1.9 * INSN_COST);
11232 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11233
11234 ins_encode %{
11235 __ sub(as_Register($dst$$reg),
11236 as_Register($src1$$reg),
11237 as_Register($src2$$reg),
11238 Assembler::LSR,
11239 $src3$$constant & 0x3f);
11240 %}
11241
11242 ins_pipe(ialu_reg_reg_shift);
11243 %}
11244
11245 // This pattern is automatically generated from aarch64_ad.m4
11246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11247 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11248 iRegIorL2I src1, iRegIorL2I src2,
11249 immI src3) %{
11250 match(Set dst (SubI src1 (RShiftI src2 src3)));
11251
11252 ins_cost(1.9 * INSN_COST);
11253 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11254
11255 ins_encode %{
11256 __ subw(as_Register($dst$$reg),
11257 as_Register($src1$$reg),
11258 as_Register($src2$$reg),
11259 Assembler::ASR,
11260 $src3$$constant & 0x1f);
11261 %}
11262
11263 ins_pipe(ialu_reg_reg_shift);
11264 %}
11265
11266 // This pattern is automatically generated from aarch64_ad.m4
11267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11268 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11269 iRegL src1, iRegL src2,
11270 immI src3) %{
11271 match(Set dst (SubL src1 (RShiftL src2 src3)));
11272
11273 ins_cost(1.9 * INSN_COST);
11274 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11275
11276 ins_encode %{
11277 __ sub(as_Register($dst$$reg),
11278 as_Register($src1$$reg),
11279 as_Register($src2$$reg),
11280 Assembler::ASR,
11281 $src3$$constant & 0x3f);
11282 %}
11283
11284 ins_pipe(ialu_reg_reg_shift);
11285 %}
11286
11287 // This pattern is automatically generated from aarch64_ad.m4
11288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11289 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11290 iRegIorL2I src1, iRegIorL2I src2,
11291 immI src3) %{
11292 match(Set dst (SubI src1 (LShiftI src2 src3)));
11293
11294 ins_cost(1.9 * INSN_COST);
11295 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11296
11297 ins_encode %{
11298 __ subw(as_Register($dst$$reg),
11299 as_Register($src1$$reg),
11300 as_Register($src2$$reg),
11301 Assembler::LSL,
11302 $src3$$constant & 0x1f);
11303 %}
11304
11305 ins_pipe(ialu_reg_reg_shift);
11306 %}
11307
11308 // This pattern is automatically generated from aarch64_ad.m4
11309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11310 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11311 iRegL src1, iRegL src2,
11312 immI src3) %{
11313 match(Set dst (SubL src1 (LShiftL src2 src3)));
11314
11315 ins_cost(1.9 * INSN_COST);
11316 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11317
11318 ins_encode %{
11319 __ sub(as_Register($dst$$reg),
11320 as_Register($src1$$reg),
11321 as_Register($src2$$reg),
11322 Assembler::LSL,
11323 $src3$$constant & 0x3f);
11324 %}
11325
11326 ins_pipe(ialu_reg_reg_shift);
11327 %}
11328
11329 // This pattern is automatically generated from aarch64_ad.m4
11330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11331
11332 // Shift Left followed by Shift Right.
11333 // This idiom is used by the compiler for the i2b bytecode etc.
11334 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11335 %{
11336 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11337 ins_cost(INSN_COST * 2);
11338 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11339 ins_encode %{
11340 int lshift = $lshift_count$$constant & 63;
11341 int rshift = $rshift_count$$constant & 63;
11342 int s = 63 - lshift;
11343 int r = (rshift - lshift) & 63;
11344 __ sbfm(as_Register($dst$$reg),
11345 as_Register($src$$reg),
11346 r, s);
11347 %}
11348
11349 ins_pipe(ialu_reg_shift);
11350 %}
11351
11352 // This pattern is automatically generated from aarch64_ad.m4
11353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11354
11355 // Shift Left followed by Shift Right.
11356 // This idiom is used by the compiler for the i2b bytecode etc.
11357 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11358 %{
11359 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11360 ins_cost(INSN_COST * 2);
11361 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11362 ins_encode %{
11363 int lshift = $lshift_count$$constant & 31;
11364 int rshift = $rshift_count$$constant & 31;
11365 int s = 31 - lshift;
11366 int r = (rshift - lshift) & 31;
11367 __ sbfmw(as_Register($dst$$reg),
11368 as_Register($src$$reg),
11369 r, s);
11370 %}
11371
11372 ins_pipe(ialu_reg_shift);
11373 %}
11374
11375 // This pattern is automatically generated from aarch64_ad.m4
11376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11377
11378 // Shift Left followed by Shift Right.
11379 // This idiom is used by the compiler for the i2b bytecode etc.
11380 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11381 %{
11382 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11383 ins_cost(INSN_COST * 2);
11384 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11385 ins_encode %{
11386 int lshift = $lshift_count$$constant & 63;
11387 int rshift = $rshift_count$$constant & 63;
11388 int s = 63 - lshift;
11389 int r = (rshift - lshift) & 63;
11390 __ ubfm(as_Register($dst$$reg),
11391 as_Register($src$$reg),
11392 r, s);
11393 %}
11394
11395 ins_pipe(ialu_reg_shift);
11396 %}
11397
11398 // This pattern is automatically generated from aarch64_ad.m4
11399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11400
11401 // Shift Left followed by Shift Right.
11402 // This idiom is used by the compiler for the i2b bytecode etc.
11403 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11404 %{
11405 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11406 ins_cost(INSN_COST * 2);
11407 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11408 ins_encode %{
11409 int lshift = $lshift_count$$constant & 31;
11410 int rshift = $rshift_count$$constant & 31;
11411 int s = 31 - lshift;
11412 int r = (rshift - lshift) & 31;
11413 __ ubfmw(as_Register($dst$$reg),
11414 as_Register($src$$reg),
11415 r, s);
11416 %}
11417
11418 ins_pipe(ialu_reg_shift);
11419 %}
11420
11421 // Bitfield extract with shift & mask
11422
11423 // This pattern is automatically generated from aarch64_ad.m4
11424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11425 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11426 %{
11427 match(Set dst (AndI (URShiftI src rshift) mask));
11428 // Make sure we are not going to exceed what ubfxw can do.
11429 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11430
11431 ins_cost(INSN_COST);
11432 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11433 ins_encode %{
11434 int rshift = $rshift$$constant & 31;
11435 intptr_t mask = $mask$$constant;
11436 int width = exact_log2(mask+1);
11437 __ ubfxw(as_Register($dst$$reg),
11438 as_Register($src$$reg), rshift, width);
11439 %}
11440 ins_pipe(ialu_reg_shift);
11441 %}
11442
11443 // This pattern is automatically generated from aarch64_ad.m4
11444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11445 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11446 %{
11447 match(Set dst (AndL (URShiftL src rshift) mask));
11448 // Make sure we are not going to exceed what ubfx can do.
11449 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11450
11451 ins_cost(INSN_COST);
11452 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11453 ins_encode %{
11454 int rshift = $rshift$$constant & 63;
11455 intptr_t mask = $mask$$constant;
11456 int width = exact_log2_long(mask+1);
11457 __ ubfx(as_Register($dst$$reg),
11458 as_Register($src$$reg), rshift, width);
11459 %}
11460 ins_pipe(ialu_reg_shift);
11461 %}
11462
11463
11464 // This pattern is automatically generated from aarch64_ad.m4
11465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11466
11467 // We can use ubfx when extending an And with a mask when we know mask
11468 // is positive. We know that because immI_bitmask guarantees it.
11469 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11470 %{
11471 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11472 // Make sure we are not going to exceed what ubfxw can do.
11473 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11474
11475 ins_cost(INSN_COST * 2);
11476 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11477 ins_encode %{
11478 int rshift = $rshift$$constant & 31;
11479 intptr_t mask = $mask$$constant;
11480 int width = exact_log2(mask+1);
11481 __ ubfx(as_Register($dst$$reg),
11482 as_Register($src$$reg), rshift, width);
11483 %}
11484 ins_pipe(ialu_reg_shift);
11485 %}
11486
11487
11488 // This pattern is automatically generated from aarch64_ad.m4
11489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11490
11491 // We can use ubfiz when masking by a positive number and then left shifting the result.
11492 // We know that the mask is positive because immI_bitmask guarantees it.
11493 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11494 %{
11495 match(Set dst (LShiftI (AndI src mask) lshift));
11496 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11497
11498 ins_cost(INSN_COST);
11499 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11500 ins_encode %{
11501 int lshift = $lshift$$constant & 31;
11502 intptr_t mask = $mask$$constant;
11503 int width = exact_log2(mask+1);
11504 __ ubfizw(as_Register($dst$$reg),
11505 as_Register($src$$reg), lshift, width);
11506 %}
11507 ins_pipe(ialu_reg_shift);
11508 %}
11509
11510 // This pattern is automatically generated from aarch64_ad.m4
11511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11512
11513 // We can use ubfiz when masking by a positive number and then left shifting the result.
11514 // We know that the mask is positive because immL_bitmask guarantees it.
11515 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11516 %{
11517 match(Set dst (LShiftL (AndL src mask) lshift));
11518 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11519
11520 ins_cost(INSN_COST);
11521 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11522 ins_encode %{
11523 int lshift = $lshift$$constant & 63;
11524 intptr_t mask = $mask$$constant;
11525 int width = exact_log2_long(mask+1);
11526 __ ubfiz(as_Register($dst$$reg),
11527 as_Register($src$$reg), lshift, width);
11528 %}
11529 ins_pipe(ialu_reg_shift);
11530 %}
11531
11532 // This pattern is automatically generated from aarch64_ad.m4
11533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11534
11535 // We can use ubfiz when masking by a positive number and then left shifting the result.
11536 // We know that the mask is positive because immI_bitmask guarantees it.
11537 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11538 %{
11539 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11540 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11541
11542 ins_cost(INSN_COST);
11543 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11544 ins_encode %{
11545 int lshift = $lshift$$constant & 31;
11546 intptr_t mask = $mask$$constant;
11547 int width = exact_log2(mask+1);
11548 __ ubfizw(as_Register($dst$$reg),
11549 as_Register($src$$reg), lshift, width);
11550 %}
11551 ins_pipe(ialu_reg_shift);
11552 %}
11553
11554 // This pattern is automatically generated from aarch64_ad.m4
11555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11556
11557 // We can use ubfiz when masking by a positive number and then left shifting the result.
11558 // We know that the mask is positive because immL_bitmask guarantees it.
11559 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11560 %{
11561 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11562 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11563
11564 ins_cost(INSN_COST);
11565 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11566 ins_encode %{
11567 int lshift = $lshift$$constant & 63;
11568 intptr_t mask = $mask$$constant;
11569 int width = exact_log2_long(mask+1);
11570 __ ubfiz(as_Register($dst$$reg),
11571 as_Register($src$$reg), lshift, width);
11572 %}
11573 ins_pipe(ialu_reg_shift);
11574 %}
11575
11576
11577 // This pattern is automatically generated from aarch64_ad.m4
11578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11579
11580 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11581 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11582 %{
11583 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11584 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11585
11586 ins_cost(INSN_COST);
11587 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11588 ins_encode %{
11589 int lshift = $lshift$$constant & 63;
11590 intptr_t mask = $mask$$constant;
11591 int width = exact_log2(mask+1);
11592 __ ubfiz(as_Register($dst$$reg),
11593 as_Register($src$$reg), lshift, width);
11594 %}
11595 ins_pipe(ialu_reg_shift);
11596 %}
11597
11598 // This pattern is automatically generated from aarch64_ad.m4
11599 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11600
11601 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11602 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11603 %{
11604 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11605 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11606
11607 ins_cost(INSN_COST);
11608 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11609 ins_encode %{
11610 int lshift = $lshift$$constant & 31;
11611 intptr_t mask = $mask$$constant;
11612 int width = exact_log2(mask+1);
11613 __ ubfiz(as_Register($dst$$reg),
11614 as_Register($src$$reg), lshift, width);
11615 %}
11616 ins_pipe(ialu_reg_shift);
11617 %}
11618
11619 // This pattern is automatically generated from aarch64_ad.m4
11620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11621
11622 // Can skip int2long conversions after AND with small bitmask
11623 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11624 %{
11625 match(Set dst (ConvI2L (AndI src msk)));
11626 ins_cost(INSN_COST);
11627 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11628 ins_encode %{
11629 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11630 %}
11631 ins_pipe(ialu_reg_shift);
11632 %}
11633
11634
11635 // Rotations
11636
11637 // This pattern is automatically generated from aarch64_ad.m4
11638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11639 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11640 %{
11641 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11642 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11643
11644 ins_cost(INSN_COST);
11645 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11646
11647 ins_encode %{
11648 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11649 $rshift$$constant & 63);
11650 %}
11651 ins_pipe(ialu_reg_reg_extr);
11652 %}
11653
11654
11655 // This pattern is automatically generated from aarch64_ad.m4
11656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11657 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11658 %{
11659 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11660 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11661
11662 ins_cost(INSN_COST);
11663 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11664
11665 ins_encode %{
11666 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11667 $rshift$$constant & 31);
11668 %}
11669 ins_pipe(ialu_reg_reg_extr);
11670 %}
11671
11672
11673 // This pattern is automatically generated from aarch64_ad.m4
11674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11675 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11676 %{
11677 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11678 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11679
11680 ins_cost(INSN_COST);
11681 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11682
11683 ins_encode %{
11684 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11685 $rshift$$constant & 63);
11686 %}
11687 ins_pipe(ialu_reg_reg_extr);
11688 %}
11689
11690
11691 // This pattern is automatically generated from aarch64_ad.m4
11692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11693 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11694 %{
11695 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11696 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11697
11698 ins_cost(INSN_COST);
11699 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11700
11701 ins_encode %{
11702 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11703 $rshift$$constant & 31);
11704 %}
11705 ins_pipe(ialu_reg_reg_extr);
11706 %}
11707
11708 // This pattern is automatically generated from aarch64_ad.m4
11709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11710 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11711 %{
11712 match(Set dst (RotateRight src shift));
11713
11714 ins_cost(INSN_COST);
11715 format %{ "ror $dst, $src, $shift" %}
11716
11717 ins_encode %{
11718 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11719 $shift$$constant & 0x1f);
11720 %}
11721 ins_pipe(ialu_reg_reg_vshift);
11722 %}
11723
11724 // This pattern is automatically generated from aarch64_ad.m4
11725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11726 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11727 %{
11728 match(Set dst (RotateRight src shift));
11729
11730 ins_cost(INSN_COST);
11731 format %{ "ror $dst, $src, $shift" %}
11732
11733 ins_encode %{
11734 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11735 $shift$$constant & 0x3f);
11736 %}
11737 ins_pipe(ialu_reg_reg_vshift);
11738 %}
11739
11740 // This pattern is automatically generated from aarch64_ad.m4
11741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11742 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11743 %{
11744 match(Set dst (RotateRight src shift));
11745
11746 ins_cost(INSN_COST);
11747 format %{ "ror $dst, $src, $shift" %}
11748
11749 ins_encode %{
11750 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11751 %}
11752 ins_pipe(ialu_reg_reg_vshift);
11753 %}
11754
11755 // This pattern is automatically generated from aarch64_ad.m4
11756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11757 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11758 %{
11759 match(Set dst (RotateRight src shift));
11760
11761 ins_cost(INSN_COST);
11762 format %{ "ror $dst, $src, $shift" %}
11763
11764 ins_encode %{
11765 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11766 %}
11767 ins_pipe(ialu_reg_reg_vshift);
11768 %}
11769
11770 // This pattern is automatically generated from aarch64_ad.m4
11771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11772 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11773 %{
11774 match(Set dst (RotateLeft src shift));
11775
11776 ins_cost(INSN_COST);
11777 format %{ "rol $dst, $src, $shift" %}
11778
11779 ins_encode %{
11780 __ subw(rscratch1, zr, as_Register($shift$$reg));
11781 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11782 %}
11783 ins_pipe(ialu_reg_reg_vshift);
11784 %}
11785
11786 // This pattern is automatically generated from aarch64_ad.m4
11787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11788 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11789 %{
11790 match(Set dst (RotateLeft src shift));
11791
11792 ins_cost(INSN_COST);
11793 format %{ "rol $dst, $src, $shift" %}
11794
11795 ins_encode %{
11796 __ subw(rscratch1, zr, as_Register($shift$$reg));
11797 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11798 %}
11799 ins_pipe(ialu_reg_reg_vshift);
11800 %}
11801
11802
11803 // Add/subtract (extended)
11804
11805 // This pattern is automatically generated from aarch64_ad.m4
11806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11807 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11808 %{
11809 match(Set dst (AddL src1 (ConvI2L src2)));
11810 ins_cost(INSN_COST);
11811 format %{ "add $dst, $src1, $src2, sxtw" %}
11812
11813 ins_encode %{
11814 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11815 as_Register($src2$$reg), ext::sxtw);
11816 %}
11817 ins_pipe(ialu_reg_reg);
11818 %}
11819
11820 // This pattern is automatically generated from aarch64_ad.m4
11821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11822 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11823 %{
11824 match(Set dst (SubL src1 (ConvI2L src2)));
11825 ins_cost(INSN_COST);
11826 format %{ "sub $dst, $src1, $src2, sxtw" %}
11827
11828 ins_encode %{
11829 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11830 as_Register($src2$$reg), ext::sxtw);
11831 %}
11832 ins_pipe(ialu_reg_reg);
11833 %}
11834
11835 // This pattern is automatically generated from aarch64_ad.m4
11836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11837 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11838 %{
11839 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11840 ins_cost(INSN_COST);
11841 format %{ "add $dst, $src1, $src2, sxth" %}
11842
11843 ins_encode %{
11844 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11845 as_Register($src2$$reg), ext::sxth);
11846 %}
11847 ins_pipe(ialu_reg_reg);
11848 %}
11849
11850 // This pattern is automatically generated from aarch64_ad.m4
11851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11852 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11853 %{
11854 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11855 ins_cost(INSN_COST);
11856 format %{ "add $dst, $src1, $src2, sxtb" %}
11857
11858 ins_encode %{
11859 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11860 as_Register($src2$$reg), ext::sxtb);
11861 %}
11862 ins_pipe(ialu_reg_reg);
11863 %}
11864
11865 // This pattern is automatically generated from aarch64_ad.m4
11866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11867 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11868 %{
11869 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11870 ins_cost(INSN_COST);
11871 format %{ "add $dst, $src1, $src2, uxtb" %}
11872
11873 ins_encode %{
11874 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11875 as_Register($src2$$reg), ext::uxtb);
11876 %}
11877 ins_pipe(ialu_reg_reg);
11878 %}
11879
11880 // This pattern is automatically generated from aarch64_ad.m4
11881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11882 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11883 %{
11884 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11885 ins_cost(INSN_COST);
11886 format %{ "add $dst, $src1, $src2, sxth" %}
11887
11888 ins_encode %{
11889 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11890 as_Register($src2$$reg), ext::sxth);
11891 %}
11892 ins_pipe(ialu_reg_reg);
11893 %}
11894
11895 // This pattern is automatically generated from aarch64_ad.m4
11896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11897 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11898 %{
11899 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11900 ins_cost(INSN_COST);
11901 format %{ "add $dst, $src1, $src2, sxtw" %}
11902
11903 ins_encode %{
11904 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11905 as_Register($src2$$reg), ext::sxtw);
11906 %}
11907 ins_pipe(ialu_reg_reg);
11908 %}
11909
11910 // This pattern is automatically generated from aarch64_ad.m4
11911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11912 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11913 %{
11914 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11915 ins_cost(INSN_COST);
11916 format %{ "add $dst, $src1, $src2, sxtb" %}
11917
11918 ins_encode %{
11919 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11920 as_Register($src2$$reg), ext::sxtb);
11921 %}
11922 ins_pipe(ialu_reg_reg);
11923 %}
11924
11925 // This pattern is automatically generated from aarch64_ad.m4
11926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11927 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11928 %{
11929 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11930 ins_cost(INSN_COST);
11931 format %{ "add $dst, $src1, $src2, uxtb" %}
11932
11933 ins_encode %{
11934 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11935 as_Register($src2$$reg), ext::uxtb);
11936 %}
11937 ins_pipe(ialu_reg_reg);
11938 %}
11939
11940 // This pattern is automatically generated from aarch64_ad.m4
11941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11942 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11943 %{
11944 match(Set dst (AddI src1 (AndI src2 mask)));
11945 ins_cost(INSN_COST);
11946 format %{ "addw $dst, $src1, $src2, uxtb" %}
11947
11948 ins_encode %{
11949 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11950 as_Register($src2$$reg), ext::uxtb);
11951 %}
11952 ins_pipe(ialu_reg_reg);
11953 %}
11954
11955 // This pattern is automatically generated from aarch64_ad.m4
11956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11957 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11958 %{
11959 match(Set dst (AddI src1 (AndI src2 mask)));
11960 ins_cost(INSN_COST);
11961 format %{ "addw $dst, $src1, $src2, uxth" %}
11962
11963 ins_encode %{
11964 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11965 as_Register($src2$$reg), ext::uxth);
11966 %}
11967 ins_pipe(ialu_reg_reg);
11968 %}
11969
11970 // This pattern is automatically generated from aarch64_ad.m4
11971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11972 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
11973 %{
11974 match(Set dst (AddL src1 (AndL src2 mask)));
11975 ins_cost(INSN_COST);
11976 format %{ "add $dst, $src1, $src2, uxtb" %}
11977
11978 ins_encode %{
11979 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11980 as_Register($src2$$reg), ext::uxtb);
11981 %}
11982 ins_pipe(ialu_reg_reg);
11983 %}
11984
11985 // This pattern is automatically generated from aarch64_ad.m4
11986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11987 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
11988 %{
11989 match(Set dst (AddL src1 (AndL src2 mask)));
11990 ins_cost(INSN_COST);
11991 format %{ "add $dst, $src1, $src2, uxth" %}
11992
11993 ins_encode %{
11994 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11995 as_Register($src2$$reg), ext::uxth);
11996 %}
11997 ins_pipe(ialu_reg_reg);
11998 %}
11999
12000 // This pattern is automatically generated from aarch64_ad.m4
12001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12002 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12003 %{
12004 match(Set dst (AddL src1 (AndL src2 mask)));
12005 ins_cost(INSN_COST);
12006 format %{ "add $dst, $src1, $src2, uxtw" %}
12007
12008 ins_encode %{
12009 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12010 as_Register($src2$$reg), ext::uxtw);
12011 %}
12012 ins_pipe(ialu_reg_reg);
12013 %}
12014
12015 // This pattern is automatically generated from aarch64_ad.m4
12016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12017 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12018 %{
12019 match(Set dst (SubI src1 (AndI src2 mask)));
12020 ins_cost(INSN_COST);
12021 format %{ "subw $dst, $src1, $src2, uxtb" %}
12022
12023 ins_encode %{
12024 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12025 as_Register($src2$$reg), ext::uxtb);
12026 %}
12027 ins_pipe(ialu_reg_reg);
12028 %}
12029
12030 // This pattern is automatically generated from aarch64_ad.m4
12031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12032 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12033 %{
12034 match(Set dst (SubI src1 (AndI src2 mask)));
12035 ins_cost(INSN_COST);
12036 format %{ "subw $dst, $src1, $src2, uxth" %}
12037
12038 ins_encode %{
12039 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12040 as_Register($src2$$reg), ext::uxth);
12041 %}
12042 ins_pipe(ialu_reg_reg);
12043 %}
12044
12045 // This pattern is automatically generated from aarch64_ad.m4
12046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12047 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12048 %{
12049 match(Set dst (SubL src1 (AndL src2 mask)));
12050 ins_cost(INSN_COST);
12051 format %{ "sub $dst, $src1, $src2, uxtb" %}
12052
12053 ins_encode %{
12054 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12055 as_Register($src2$$reg), ext::uxtb);
12056 %}
12057 ins_pipe(ialu_reg_reg);
12058 %}
12059
12060 // This pattern is automatically generated from aarch64_ad.m4
12061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12062 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12063 %{
12064 match(Set dst (SubL src1 (AndL src2 mask)));
12065 ins_cost(INSN_COST);
12066 format %{ "sub $dst, $src1, $src2, uxth" %}
12067
12068 ins_encode %{
12069 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12070 as_Register($src2$$reg), ext::uxth);
12071 %}
12072 ins_pipe(ialu_reg_reg);
12073 %}
12074
12075 // This pattern is automatically generated from aarch64_ad.m4
12076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12077 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12078 %{
12079 match(Set dst (SubL src1 (AndL src2 mask)));
12080 ins_cost(INSN_COST);
12081 format %{ "sub $dst, $src1, $src2, uxtw" %}
12082
12083 ins_encode %{
12084 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12085 as_Register($src2$$reg), ext::uxtw);
12086 %}
12087 ins_pipe(ialu_reg_reg);
12088 %}
12089
12090
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12094 %{
12095 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12096 ins_cost(1.9 * INSN_COST);
12097 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12098
12099 ins_encode %{
12100 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12101 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12102 %}
12103 ins_pipe(ialu_reg_reg_shift);
12104 %}
12105
12106 // This pattern is automatically generated from aarch64_ad.m4
12107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12108 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12109 %{
12110 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12111 ins_cost(1.9 * INSN_COST);
12112 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12113
12114 ins_encode %{
12115 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12116 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12117 %}
12118 ins_pipe(ialu_reg_reg_shift);
12119 %}
12120
12121 // This pattern is automatically generated from aarch64_ad.m4
12122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12123 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12124 %{
12125 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12126 ins_cost(1.9 * INSN_COST);
12127 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12128
12129 ins_encode %{
12130 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12131 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12132 %}
12133 ins_pipe(ialu_reg_reg_shift);
12134 %}
12135
12136 // This pattern is automatically generated from aarch64_ad.m4
12137 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12138 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12139 %{
12140 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12141 ins_cost(1.9 * INSN_COST);
12142 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12143
12144 ins_encode %{
12145 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12146 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12147 %}
12148 ins_pipe(ialu_reg_reg_shift);
12149 %}
12150
12151 // This pattern is automatically generated from aarch64_ad.m4
12152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12153 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12154 %{
12155 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12156 ins_cost(1.9 * INSN_COST);
12157 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12158
12159 ins_encode %{
12160 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12161 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12162 %}
12163 ins_pipe(ialu_reg_reg_shift);
12164 %}
12165
12166 // This pattern is automatically generated from aarch64_ad.m4
12167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12168 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12169 %{
12170 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12171 ins_cost(1.9 * INSN_COST);
12172 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12173
12174 ins_encode %{
12175 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12176 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12177 %}
12178 ins_pipe(ialu_reg_reg_shift);
12179 %}
12180
12181 // This pattern is automatically generated from aarch64_ad.m4
12182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12183 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12184 %{
12185 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12186 ins_cost(1.9 * INSN_COST);
12187 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12188
12189 ins_encode %{
12190 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12191 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12192 %}
12193 ins_pipe(ialu_reg_reg_shift);
12194 %}
12195
12196 // This pattern is automatically generated from aarch64_ad.m4
12197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12198 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12199 %{
12200 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12201 ins_cost(1.9 * INSN_COST);
12202 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12203
12204 ins_encode %{
12205 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12206 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12207 %}
12208 ins_pipe(ialu_reg_reg_shift);
12209 %}
12210
12211 // This pattern is automatically generated from aarch64_ad.m4
12212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12213 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12214 %{
12215 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12216 ins_cost(1.9 * INSN_COST);
12217 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12218
12219 ins_encode %{
12220 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12221 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12222 %}
12223 ins_pipe(ialu_reg_reg_shift);
12224 %}
12225
12226 // This pattern is automatically generated from aarch64_ad.m4
12227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12228 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12229 %{
12230 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12231 ins_cost(1.9 * INSN_COST);
12232 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12233
12234 ins_encode %{
12235 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12236 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12237 %}
12238 ins_pipe(ialu_reg_reg_shift);
12239 %}
12240
12241 // This pattern is automatically generated from aarch64_ad.m4
12242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12243 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12244 %{
12245 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12246 ins_cost(1.9 * INSN_COST);
12247 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12248
12249 ins_encode %{
12250 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12251 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12252 %}
12253 ins_pipe(ialu_reg_reg_shift);
12254 %}
12255
12256 // This pattern is automatically generated from aarch64_ad.m4
12257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12258 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12259 %{
12260 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12261 ins_cost(1.9 * INSN_COST);
12262 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12263
12264 ins_encode %{
12265 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12266 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12267 %}
12268 ins_pipe(ialu_reg_reg_shift);
12269 %}
12270
12271 // This pattern is automatically generated from aarch64_ad.m4
12272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12273 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12274 %{
12275 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12276 ins_cost(1.9 * INSN_COST);
12277 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12278
12279 ins_encode %{
12280 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12281 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12282 %}
12283 ins_pipe(ialu_reg_reg_shift);
12284 %}
12285
12286 // This pattern is automatically generated from aarch64_ad.m4
12287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12288 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12289 %{
12290 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12291 ins_cost(1.9 * INSN_COST);
12292 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12293
12294 ins_encode %{
12295 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12296 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12297 %}
12298 ins_pipe(ialu_reg_reg_shift);
12299 %}
12300
12301 // This pattern is automatically generated from aarch64_ad.m4
12302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12303 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12304 %{
12305 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12306 ins_cost(1.9 * INSN_COST);
12307 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12308
12309 ins_encode %{
12310 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12311 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12312 %}
12313 ins_pipe(ialu_reg_reg_shift);
12314 %}
12315
12316 // This pattern is automatically generated from aarch64_ad.m4
12317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12318 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12319 %{
12320 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12321 ins_cost(1.9 * INSN_COST);
12322 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12323
12324 ins_encode %{
12325 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12326 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12327 %}
12328 ins_pipe(ialu_reg_reg_shift);
12329 %}
12330
12331 // This pattern is automatically generated from aarch64_ad.m4
12332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12333 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12334 %{
12335 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12336 ins_cost(1.9 * INSN_COST);
12337 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12338
12339 ins_encode %{
12340 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12341 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12342 %}
12343 ins_pipe(ialu_reg_reg_shift);
12344 %}
12345
12346 // This pattern is automatically generated from aarch64_ad.m4
12347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12348 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12349 %{
12350 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12351 ins_cost(1.9 * INSN_COST);
12352 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12353
12354 ins_encode %{
12355 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12356 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12357 %}
12358 ins_pipe(ialu_reg_reg_shift);
12359 %}
12360
12361 // This pattern is automatically generated from aarch64_ad.m4
12362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12363 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12364 %{
12365 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12366 ins_cost(1.9 * INSN_COST);
12367 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12368
12369 ins_encode %{
12370 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12371 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12372 %}
12373 ins_pipe(ialu_reg_reg_shift);
12374 %}
12375
12376 // This pattern is automatically generated from aarch64_ad.m4
12377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12378 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12379 %{
12380 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12381 ins_cost(1.9 * INSN_COST);
12382 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12383
12384 ins_encode %{
12385 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12386 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12387 %}
12388 ins_pipe(ialu_reg_reg_shift);
12389 %}
12390
12391 // This pattern is automatically generated from aarch64_ad.m4
12392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12393 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12394 %{
12395 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12396 ins_cost(1.9 * INSN_COST);
12397 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12398
12399 ins_encode %{
12400 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12401 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12402 %}
12403 ins_pipe(ialu_reg_reg_shift);
12404 %}
12405
12406 // This pattern is automatically generated from aarch64_ad.m4
12407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12408 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12409 %{
12410 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12411 ins_cost(1.9 * INSN_COST);
12412 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12413
12414 ins_encode %{
12415 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12416 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12417 %}
12418 ins_pipe(ialu_reg_reg_shift);
12419 %}
12420
12421 // This pattern is automatically generated from aarch64_ad.m4
12422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12423 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12424 %{
12425 effect(DEF dst, USE src1, USE src2, USE cr);
12426 ins_cost(INSN_COST * 2);
12427 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12428
12429 ins_encode %{
12430 __ cselw($dst$$Register,
12431 $src1$$Register,
12432 $src2$$Register,
12433 Assembler::LT);
12434 %}
12435 ins_pipe(icond_reg_reg);
12436 %}
12437
12438 // This pattern is automatically generated from aarch64_ad.m4
12439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12440 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12441 %{
12442 effect(DEF dst, USE src1, USE src2, USE cr);
12443 ins_cost(INSN_COST * 2);
12444 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12445
12446 ins_encode %{
12447 __ cselw($dst$$Register,
12448 $src1$$Register,
12449 $src2$$Register,
12450 Assembler::GT);
12451 %}
12452 ins_pipe(icond_reg_reg);
12453 %}
12454
12455 // This pattern is automatically generated from aarch64_ad.m4
12456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12457 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12458 %{
12459 effect(DEF dst, USE src1, USE cr);
12460 ins_cost(INSN_COST * 2);
12461 format %{ "cselw $dst, $src1, zr lt\t" %}
12462
12463 ins_encode %{
12464 __ cselw($dst$$Register,
12465 $src1$$Register,
12466 zr,
12467 Assembler::LT);
12468 %}
12469 ins_pipe(icond_reg);
12470 %}
12471
12472 // This pattern is automatically generated from aarch64_ad.m4
12473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12474 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12475 %{
12476 effect(DEF dst, USE src1, USE cr);
12477 ins_cost(INSN_COST * 2);
12478 format %{ "cselw $dst, $src1, zr gt\t" %}
12479
12480 ins_encode %{
12481 __ cselw($dst$$Register,
12482 $src1$$Register,
12483 zr,
12484 Assembler::GT);
12485 %}
12486 ins_pipe(icond_reg);
12487 %}
12488
12489 // This pattern is automatically generated from aarch64_ad.m4
12490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12491 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12492 %{
12493 effect(DEF dst, USE src1, USE cr);
12494 ins_cost(INSN_COST * 2);
12495 format %{ "csincw $dst, $src1, zr le\t" %}
12496
12497 ins_encode %{
12498 __ csincw($dst$$Register,
12499 $src1$$Register,
12500 zr,
12501 Assembler::LE);
12502 %}
12503 ins_pipe(icond_reg);
12504 %}
12505
12506 // This pattern is automatically generated from aarch64_ad.m4
12507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12508 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12509 %{
12510 effect(DEF dst, USE src1, USE cr);
12511 ins_cost(INSN_COST * 2);
12512 format %{ "csincw $dst, $src1, zr gt\t" %}
12513
12514 ins_encode %{
12515 __ csincw($dst$$Register,
12516 $src1$$Register,
12517 zr,
12518 Assembler::GT);
12519 %}
12520 ins_pipe(icond_reg);
12521 %}
12522
12523 // This pattern is automatically generated from aarch64_ad.m4
12524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12525 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12526 %{
12527 effect(DEF dst, USE src1, USE cr);
12528 ins_cost(INSN_COST * 2);
12529 format %{ "csinvw $dst, $src1, zr lt\t" %}
12530
12531 ins_encode %{
12532 __ csinvw($dst$$Register,
12533 $src1$$Register,
12534 zr,
12535 Assembler::LT);
12536 %}
12537 ins_pipe(icond_reg);
12538 %}
12539
12540 // This pattern is automatically generated from aarch64_ad.m4
12541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12542 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12543 %{
12544 effect(DEF dst, USE src1, USE cr);
12545 ins_cost(INSN_COST * 2);
12546 format %{ "csinvw $dst, $src1, zr ge\t" %}
12547
12548 ins_encode %{
12549 __ csinvw($dst$$Register,
12550 $src1$$Register,
12551 zr,
12552 Assembler::GE);
12553 %}
12554 ins_pipe(icond_reg);
12555 %}
12556
12557 // This pattern is automatically generated from aarch64_ad.m4
12558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12559 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12560 %{
12561 match(Set dst (MinI src imm));
12562 ins_cost(INSN_COST * 3);
12563 expand %{
12564 rFlagsReg cr;
12565 compI_reg_imm0(cr, src);
12566 cmovI_reg_imm0_lt(dst, src, cr);
12567 %}
12568 %}
12569
12570 // This pattern is automatically generated from aarch64_ad.m4
12571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12572 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12573 %{
12574 match(Set dst (MinI imm src));
12575 ins_cost(INSN_COST * 3);
12576 expand %{
12577 rFlagsReg cr;
12578 compI_reg_imm0(cr, src);
12579 cmovI_reg_imm0_lt(dst, src, cr);
12580 %}
12581 %}
12582
12583 // This pattern is automatically generated from aarch64_ad.m4
12584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12585 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12586 %{
12587 match(Set dst (MinI src imm));
12588 ins_cost(INSN_COST * 3);
12589 expand %{
12590 rFlagsReg cr;
12591 compI_reg_imm0(cr, src);
12592 cmovI_reg_imm1_le(dst, src, cr);
12593 %}
12594 %}
12595
12596 // This pattern is automatically generated from aarch64_ad.m4
12597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12598 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12599 %{
12600 match(Set dst (MinI imm src));
12601 ins_cost(INSN_COST * 3);
12602 expand %{
12603 rFlagsReg cr;
12604 compI_reg_imm0(cr, src);
12605 cmovI_reg_imm1_le(dst, src, cr);
12606 %}
12607 %}
12608
12609 // This pattern is automatically generated from aarch64_ad.m4
12610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12611 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12612 %{
12613 match(Set dst (MinI src imm));
12614 ins_cost(INSN_COST * 3);
12615 expand %{
12616 rFlagsReg cr;
12617 compI_reg_imm0(cr, src);
12618 cmovI_reg_immM1_lt(dst, src, cr);
12619 %}
12620 %}
12621
12622 // This pattern is automatically generated from aarch64_ad.m4
12623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12624 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12625 %{
12626 match(Set dst (MinI imm src));
12627 ins_cost(INSN_COST * 3);
12628 expand %{
12629 rFlagsReg cr;
12630 compI_reg_imm0(cr, src);
12631 cmovI_reg_immM1_lt(dst, src, cr);
12632 %}
12633 %}
12634
12635 // This pattern is automatically generated from aarch64_ad.m4
12636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12637 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12638 %{
12639 match(Set dst (MaxI src imm));
12640 ins_cost(INSN_COST * 3);
12641 expand %{
12642 rFlagsReg cr;
12643 compI_reg_imm0(cr, src);
12644 cmovI_reg_imm0_gt(dst, src, cr);
12645 %}
12646 %}
12647
12648 // This pattern is automatically generated from aarch64_ad.m4
12649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12650 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12651 %{
12652 match(Set dst (MaxI imm src));
12653 ins_cost(INSN_COST * 3);
12654 expand %{
12655 rFlagsReg cr;
12656 compI_reg_imm0(cr, src);
12657 cmovI_reg_imm0_gt(dst, src, cr);
12658 %}
12659 %}
12660
12661 // This pattern is automatically generated from aarch64_ad.m4
12662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12663 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12664 %{
12665 match(Set dst (MaxI src imm));
12666 ins_cost(INSN_COST * 3);
12667 expand %{
12668 rFlagsReg cr;
12669 compI_reg_imm0(cr, src);
12670 cmovI_reg_imm1_gt(dst, src, cr);
12671 %}
12672 %}
12673
12674 // This pattern is automatically generated from aarch64_ad.m4
12675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12676 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12677 %{
12678 match(Set dst (MaxI imm src));
12679 ins_cost(INSN_COST * 3);
12680 expand %{
12681 rFlagsReg cr;
12682 compI_reg_imm0(cr, src);
12683 cmovI_reg_imm1_gt(dst, src, cr);
12684 %}
12685 %}
12686
12687 // This pattern is automatically generated from aarch64_ad.m4
12688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12689 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12690 %{
12691 match(Set dst (MaxI src imm));
12692 ins_cost(INSN_COST * 3);
12693 expand %{
12694 rFlagsReg cr;
12695 compI_reg_imm0(cr, src);
12696 cmovI_reg_immM1_ge(dst, src, cr);
12697 %}
12698 %}
12699
12700 // This pattern is automatically generated from aarch64_ad.m4
12701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12702 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12703 %{
12704 match(Set dst (MaxI imm src));
12705 ins_cost(INSN_COST * 3);
12706 expand %{
12707 rFlagsReg cr;
12708 compI_reg_imm0(cr, src);
12709 cmovI_reg_immM1_ge(dst, src, cr);
12710 %}
12711 %}
12712
12713 // This pattern is automatically generated from aarch64_ad.m4
12714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12715 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12716 %{
12717 match(Set dst (ReverseI src));
12718 ins_cost(INSN_COST);
12719 format %{ "rbitw $dst, $src" %}
12720 ins_encode %{
12721 __ rbitw($dst$$Register, $src$$Register);
12722 %}
12723 ins_pipe(ialu_reg);
12724 %}
12725
12726 // This pattern is automatically generated from aarch64_ad.m4
12727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12728 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12729 %{
12730 match(Set dst (ReverseL src));
12731 ins_cost(INSN_COST);
12732 format %{ "rbit $dst, $src" %}
12733 ins_encode %{
12734 __ rbit($dst$$Register, $src$$Register);
12735 %}
12736 ins_pipe(ialu_reg);
12737 %}
12738
12739
12740 // END This section of the file is automatically generated. Do not edit --------------
12741
12742
12743 // ============================================================================
12744 // Floating Point Arithmetic Instructions
12745
12746 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12747 match(Set dst (AddHF src1 src2));
12748 format %{ "faddh $dst, $src1, $src2" %}
12749 ins_encode %{
12750 __ faddh($dst$$FloatRegister,
12751 $src1$$FloatRegister,
12752 $src2$$FloatRegister);
12753 %}
12754 ins_pipe(fp_dop_reg_reg_s);
12755 %}
12756
12757 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12758 match(Set dst (AddF src1 src2));
12759
12760 ins_cost(INSN_COST * 5);
12761 format %{ "fadds $dst, $src1, $src2" %}
12762
12763 ins_encode %{
12764 __ fadds(as_FloatRegister($dst$$reg),
12765 as_FloatRegister($src1$$reg),
12766 as_FloatRegister($src2$$reg));
12767 %}
12768
12769 ins_pipe(fp_dop_reg_reg_s);
12770 %}
12771
12772 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12773 match(Set dst (AddD src1 src2));
12774
12775 ins_cost(INSN_COST * 5);
12776 format %{ "faddd $dst, $src1, $src2" %}
12777
12778 ins_encode %{
12779 __ faddd(as_FloatRegister($dst$$reg),
12780 as_FloatRegister($src1$$reg),
12781 as_FloatRegister($src2$$reg));
12782 %}
12783
12784 ins_pipe(fp_dop_reg_reg_d);
12785 %}
12786
12787 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12788 match(Set dst (SubHF src1 src2));
12789 format %{ "fsubh $dst, $src1, $src2" %}
12790 ins_encode %{
12791 __ fsubh($dst$$FloatRegister,
12792 $src1$$FloatRegister,
12793 $src2$$FloatRegister);
12794 %}
12795 ins_pipe(fp_dop_reg_reg_s);
12796 %}
12797
12798 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12799 match(Set dst (SubF src1 src2));
12800
12801 ins_cost(INSN_COST * 5);
12802 format %{ "fsubs $dst, $src1, $src2" %}
12803
12804 ins_encode %{
12805 __ fsubs(as_FloatRegister($dst$$reg),
12806 as_FloatRegister($src1$$reg),
12807 as_FloatRegister($src2$$reg));
12808 %}
12809
12810 ins_pipe(fp_dop_reg_reg_s);
12811 %}
12812
12813 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12814 match(Set dst (SubD src1 src2));
12815
12816 ins_cost(INSN_COST * 5);
12817 format %{ "fsubd $dst, $src1, $src2" %}
12818
12819 ins_encode %{
12820 __ fsubd(as_FloatRegister($dst$$reg),
12821 as_FloatRegister($src1$$reg),
12822 as_FloatRegister($src2$$reg));
12823 %}
12824
12825 ins_pipe(fp_dop_reg_reg_d);
12826 %}
12827
12828 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12829 match(Set dst (MulHF src1 src2));
12830 format %{ "fmulh $dst, $src1, $src2" %}
12831 ins_encode %{
12832 __ fmulh($dst$$FloatRegister,
12833 $src1$$FloatRegister,
12834 $src2$$FloatRegister);
12835 %}
12836 ins_pipe(fp_dop_reg_reg_s);
12837 %}
12838
12839 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12840 match(Set dst (MulF src1 src2));
12841
12842 ins_cost(INSN_COST * 6);
12843 format %{ "fmuls $dst, $src1, $src2" %}
12844
12845 ins_encode %{
12846 __ fmuls(as_FloatRegister($dst$$reg),
12847 as_FloatRegister($src1$$reg),
12848 as_FloatRegister($src2$$reg));
12849 %}
12850
12851 ins_pipe(fp_dop_reg_reg_s);
12852 %}
12853
12854 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12855 match(Set dst (MulD src1 src2));
12856
12857 ins_cost(INSN_COST * 6);
12858 format %{ "fmuld $dst, $src1, $src2" %}
12859
12860 ins_encode %{
12861 __ fmuld(as_FloatRegister($dst$$reg),
12862 as_FloatRegister($src1$$reg),
12863 as_FloatRegister($src2$$reg));
12864 %}
12865
12866 ins_pipe(fp_dop_reg_reg_d);
12867 %}
12868
12869 // src1 * src2 + src3 (half-precision float)
12870 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12871 match(Set dst (FmaHF src3 (Binary src1 src2)));
12872 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12873 ins_encode %{
12874 assert(UseFMA, "Needs FMA instructions support.");
12875 __ fmaddh($dst$$FloatRegister,
12876 $src1$$FloatRegister,
12877 $src2$$FloatRegister,
12878 $src3$$FloatRegister);
12879 %}
12880 ins_pipe(pipe_class_default);
12881 %}
12882
12883 // src1 * src2 + src3
12884 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12885 match(Set dst (FmaF src3 (Binary src1 src2)));
12886
12887 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12888
12889 ins_encode %{
12890 assert(UseFMA, "Needs FMA instructions support.");
12891 __ fmadds(as_FloatRegister($dst$$reg),
12892 as_FloatRegister($src1$$reg),
12893 as_FloatRegister($src2$$reg),
12894 as_FloatRegister($src3$$reg));
12895 %}
12896
12897 ins_pipe(pipe_class_default);
12898 %}
12899
12900 // src1 * src2 + src3
12901 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12902 match(Set dst (FmaD src3 (Binary src1 src2)));
12903
12904 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12905
12906 ins_encode %{
12907 assert(UseFMA, "Needs FMA instructions support.");
12908 __ fmaddd(as_FloatRegister($dst$$reg),
12909 as_FloatRegister($src1$$reg),
12910 as_FloatRegister($src2$$reg),
12911 as_FloatRegister($src3$$reg));
12912 %}
12913
12914 ins_pipe(pipe_class_default);
12915 %}
12916
12917 // src1 * (-src2) + src3
12918 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12919 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12920 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12921
12922 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12923
12924 ins_encode %{
12925 assert(UseFMA, "Needs FMA instructions support.");
12926 __ fmsubs(as_FloatRegister($dst$$reg),
12927 as_FloatRegister($src1$$reg),
12928 as_FloatRegister($src2$$reg),
12929 as_FloatRegister($src3$$reg));
12930 %}
12931
12932 ins_pipe(pipe_class_default);
12933 %}
12934
12935 // src1 * (-src2) + src3
12936 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12937 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12938 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12939
12940 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12941
12942 ins_encode %{
12943 assert(UseFMA, "Needs FMA instructions support.");
12944 __ fmsubd(as_FloatRegister($dst$$reg),
12945 as_FloatRegister($src1$$reg),
12946 as_FloatRegister($src2$$reg),
12947 as_FloatRegister($src3$$reg));
12948 %}
12949
12950 ins_pipe(pipe_class_default);
12951 %}
12952
12953 // src1 * (-src2) - src3
12954 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
12955 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12956 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
12957
12958 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
12959
12960 ins_encode %{
12961 assert(UseFMA, "Needs FMA instructions support.");
12962 __ fnmadds(as_FloatRegister($dst$$reg),
12963 as_FloatRegister($src1$$reg),
12964 as_FloatRegister($src2$$reg),
12965 as_FloatRegister($src3$$reg));
12966 %}
12967
12968 ins_pipe(pipe_class_default);
12969 %}
12970
12971 // src1 * (-src2) - src3
12972 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
12973 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12974 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
12975
12976 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
12977
12978 ins_encode %{
12979 assert(UseFMA, "Needs FMA instructions support.");
12980 __ fnmaddd(as_FloatRegister($dst$$reg),
12981 as_FloatRegister($src1$$reg),
12982 as_FloatRegister($src2$$reg),
12983 as_FloatRegister($src3$$reg));
12984 %}
12985
12986 ins_pipe(pipe_class_default);
12987 %}
12988
12989 // src1 * src2 - src3
12990 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
12991 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
12992
12993 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
12994
12995 ins_encode %{
12996 assert(UseFMA, "Needs FMA instructions support.");
12997 __ fnmsubs(as_FloatRegister($dst$$reg),
12998 as_FloatRegister($src1$$reg),
12999 as_FloatRegister($src2$$reg),
13000 as_FloatRegister($src3$$reg));
13001 %}
13002
13003 ins_pipe(pipe_class_default);
13004 %}
13005
13006 // src1 * src2 - src3
13007 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13008 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13009
13010 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13011
13012 ins_encode %{
13013 assert(UseFMA, "Needs FMA instructions support.");
13014 // n.b. insn name should be fnmsubd
13015 __ fnmsub(as_FloatRegister($dst$$reg),
13016 as_FloatRegister($src1$$reg),
13017 as_FloatRegister($src2$$reg),
13018 as_FloatRegister($src3$$reg));
13019 %}
13020
13021 ins_pipe(pipe_class_default);
13022 %}
13023
13024 // Math.max(HH)H (half-precision float)
13025 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13026 match(Set dst (MaxHF src1 src2));
13027 format %{ "fmaxh $dst, $src1, $src2" %}
13028 ins_encode %{
13029 __ fmaxh($dst$$FloatRegister,
13030 $src1$$FloatRegister,
13031 $src2$$FloatRegister);
13032 %}
13033 ins_pipe(fp_dop_reg_reg_s);
13034 %}
13035
13036 // Math.min(HH)H (half-precision float)
13037 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13038 match(Set dst (MinHF src1 src2));
13039 format %{ "fminh $dst, $src1, $src2" %}
13040 ins_encode %{
13041 __ fminh($dst$$FloatRegister,
13042 $src1$$FloatRegister,
13043 $src2$$FloatRegister);
13044 %}
13045 ins_pipe(fp_dop_reg_reg_s);
13046 %}
13047
13048 // Math.max(FF)F
13049 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13050 match(Set dst (MaxF src1 src2));
13051
13052 format %{ "fmaxs $dst, $src1, $src2" %}
13053 ins_encode %{
13054 __ fmaxs(as_FloatRegister($dst$$reg),
13055 as_FloatRegister($src1$$reg),
13056 as_FloatRegister($src2$$reg));
13057 %}
13058
13059 ins_pipe(fp_dop_reg_reg_s);
13060 %}
13061
13062 // Math.min(FF)F
13063 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13064 match(Set dst (MinF src1 src2));
13065
13066 format %{ "fmins $dst, $src1, $src2" %}
13067 ins_encode %{
13068 __ fmins(as_FloatRegister($dst$$reg),
13069 as_FloatRegister($src1$$reg),
13070 as_FloatRegister($src2$$reg));
13071 %}
13072
13073 ins_pipe(fp_dop_reg_reg_s);
13074 %}
13075
13076 // Math.max(DD)D
13077 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13078 match(Set dst (MaxD src1 src2));
13079
13080 format %{ "fmaxd $dst, $src1, $src2" %}
13081 ins_encode %{
13082 __ fmaxd(as_FloatRegister($dst$$reg),
13083 as_FloatRegister($src1$$reg),
13084 as_FloatRegister($src2$$reg));
13085 %}
13086
13087 ins_pipe(fp_dop_reg_reg_d);
13088 %}
13089
13090 // Math.min(DD)D
13091 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13092 match(Set dst (MinD src1 src2));
13093
13094 format %{ "fmind $dst, $src1, $src2" %}
13095 ins_encode %{
13096 __ fmind(as_FloatRegister($dst$$reg),
13097 as_FloatRegister($src1$$reg),
13098 as_FloatRegister($src2$$reg));
13099 %}
13100
13101 ins_pipe(fp_dop_reg_reg_d);
13102 %}
13103
13104 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13105 match(Set dst (DivHF src1 src2));
13106 format %{ "fdivh $dst, $src1, $src2" %}
13107 ins_encode %{
13108 __ fdivh($dst$$FloatRegister,
13109 $src1$$FloatRegister,
13110 $src2$$FloatRegister);
13111 %}
13112 ins_pipe(fp_div_s);
13113 %}
13114
13115 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13116 match(Set dst (DivF src1 src2));
13117
13118 ins_cost(INSN_COST * 18);
13119 format %{ "fdivs $dst, $src1, $src2" %}
13120
13121 ins_encode %{
13122 __ fdivs(as_FloatRegister($dst$$reg),
13123 as_FloatRegister($src1$$reg),
13124 as_FloatRegister($src2$$reg));
13125 %}
13126
13127 ins_pipe(fp_div_s);
13128 %}
13129
13130 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13131 match(Set dst (DivD src1 src2));
13132
13133 ins_cost(INSN_COST * 32);
13134 format %{ "fdivd $dst, $src1, $src2" %}
13135
13136 ins_encode %{
13137 __ fdivd(as_FloatRegister($dst$$reg),
13138 as_FloatRegister($src1$$reg),
13139 as_FloatRegister($src2$$reg));
13140 %}
13141
13142 ins_pipe(fp_div_d);
13143 %}
13144
13145 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13146 match(Set dst (NegF src));
13147
13148 ins_cost(INSN_COST * 3);
13149 format %{ "fneg $dst, $src" %}
13150
13151 ins_encode %{
13152 __ fnegs(as_FloatRegister($dst$$reg),
13153 as_FloatRegister($src$$reg));
13154 %}
13155
13156 ins_pipe(fp_uop_s);
13157 %}
13158
13159 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13160 match(Set dst (NegD src));
13161
13162 ins_cost(INSN_COST * 3);
13163 format %{ "fnegd $dst, $src" %}
13164
13165 ins_encode %{
13166 __ fnegd(as_FloatRegister($dst$$reg),
13167 as_FloatRegister($src$$reg));
13168 %}
13169
13170 ins_pipe(fp_uop_d);
13171 %}
13172
13173 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13174 %{
13175 match(Set dst (AbsI src));
13176
13177 effect(KILL cr);
13178 ins_cost(INSN_COST * 2);
13179 format %{ "cmpw $src, zr\n\t"
13180 "cnegw $dst, $src, Assembler::LT\t# int abs"
13181 %}
13182
13183 ins_encode %{
13184 __ cmpw(as_Register($src$$reg), zr);
13185 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13186 %}
13187 ins_pipe(pipe_class_default);
13188 %}
13189
13190 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13191 %{
13192 match(Set dst (AbsL src));
13193
13194 effect(KILL cr);
13195 ins_cost(INSN_COST * 2);
13196 format %{ "cmp $src, zr\n\t"
13197 "cneg $dst, $src, Assembler::LT\t# long abs"
13198 %}
13199
13200 ins_encode %{
13201 __ cmp(as_Register($src$$reg), zr);
13202 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13203 %}
13204 ins_pipe(pipe_class_default);
13205 %}
13206
13207 instruct absF_reg(vRegF dst, vRegF src) %{
13208 match(Set dst (AbsF src));
13209
13210 ins_cost(INSN_COST * 3);
13211 format %{ "fabss $dst, $src" %}
13212 ins_encode %{
13213 __ fabss(as_FloatRegister($dst$$reg),
13214 as_FloatRegister($src$$reg));
13215 %}
13216
13217 ins_pipe(fp_uop_s);
13218 %}
13219
13220 instruct absD_reg(vRegD dst, vRegD src) %{
13221 match(Set dst (AbsD src));
13222
13223 ins_cost(INSN_COST * 3);
13224 format %{ "fabsd $dst, $src" %}
13225 ins_encode %{
13226 __ fabsd(as_FloatRegister($dst$$reg),
13227 as_FloatRegister($src$$reg));
13228 %}
13229
13230 ins_pipe(fp_uop_d);
13231 %}
13232
13233 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13234 match(Set dst (AbsF (SubF src1 src2)));
13235
13236 ins_cost(INSN_COST * 3);
13237 format %{ "fabds $dst, $src1, $src2" %}
13238 ins_encode %{
13239 __ fabds(as_FloatRegister($dst$$reg),
13240 as_FloatRegister($src1$$reg),
13241 as_FloatRegister($src2$$reg));
13242 %}
13243
13244 ins_pipe(fp_uop_s);
13245 %}
13246
13247 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13248 match(Set dst (AbsD (SubD src1 src2)));
13249
13250 ins_cost(INSN_COST * 3);
13251 format %{ "fabdd $dst, $src1, $src2" %}
13252 ins_encode %{
13253 __ fabdd(as_FloatRegister($dst$$reg),
13254 as_FloatRegister($src1$$reg),
13255 as_FloatRegister($src2$$reg));
13256 %}
13257
13258 ins_pipe(fp_uop_d);
13259 %}
13260
13261 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13262 match(Set dst (SqrtD src));
13263
13264 ins_cost(INSN_COST * 50);
13265 format %{ "fsqrtd $dst, $src" %}
13266 ins_encode %{
13267 __ fsqrtd(as_FloatRegister($dst$$reg),
13268 as_FloatRegister($src$$reg));
13269 %}
13270
13271 ins_pipe(fp_div_s);
13272 %}
13273
13274 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13275 match(Set dst (SqrtF src));
13276
13277 ins_cost(INSN_COST * 50);
13278 format %{ "fsqrts $dst, $src" %}
13279 ins_encode %{
13280 __ fsqrts(as_FloatRegister($dst$$reg),
13281 as_FloatRegister($src$$reg));
13282 %}
13283
13284 ins_pipe(fp_div_d);
13285 %}
13286
13287 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13288 match(Set dst (SqrtHF src));
13289 format %{ "fsqrth $dst, $src" %}
13290 ins_encode %{
13291 __ fsqrth($dst$$FloatRegister,
13292 $src$$FloatRegister);
13293 %}
13294 ins_pipe(fp_div_s);
13295 %}
13296
13297 // Math.rint, floor, ceil
13298 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13299 match(Set dst (RoundDoubleMode src rmode));
13300 format %{ "frint $dst, $src, $rmode" %}
13301 ins_encode %{
13302 switch ($rmode$$constant) {
13303 case RoundDoubleModeNode::rmode_rint:
13304 __ frintnd(as_FloatRegister($dst$$reg),
13305 as_FloatRegister($src$$reg));
13306 break;
13307 case RoundDoubleModeNode::rmode_floor:
13308 __ frintmd(as_FloatRegister($dst$$reg),
13309 as_FloatRegister($src$$reg));
13310 break;
13311 case RoundDoubleModeNode::rmode_ceil:
13312 __ frintpd(as_FloatRegister($dst$$reg),
13313 as_FloatRegister($src$$reg));
13314 break;
13315 }
13316 %}
13317 ins_pipe(fp_uop_d);
13318 %}
13319
13320 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13321 match(Set dst (CopySignD src1 (Binary src2 zero)));
13322 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13323 format %{ "CopySignD $dst $src1 $src2" %}
13324 ins_encode %{
13325 FloatRegister dst = as_FloatRegister($dst$$reg),
13326 src1 = as_FloatRegister($src1$$reg),
13327 src2 = as_FloatRegister($src2$$reg),
13328 zero = as_FloatRegister($zero$$reg);
13329 __ fnegd(dst, zero);
13330 __ bsl(dst, __ T8B, src2, src1);
13331 %}
13332 ins_pipe(fp_uop_d);
13333 %}
13334
13335 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13336 match(Set dst (CopySignF src1 src2));
13337 effect(TEMP_DEF dst, USE src1, USE src2);
13338 format %{ "CopySignF $dst $src1 $src2" %}
13339 ins_encode %{
13340 FloatRegister dst = as_FloatRegister($dst$$reg),
13341 src1 = as_FloatRegister($src1$$reg),
13342 src2 = as_FloatRegister($src2$$reg);
13343 __ movi(dst, __ T2S, 0x80, 24);
13344 __ bsl(dst, __ T8B, src2, src1);
13345 %}
13346 ins_pipe(fp_uop_d);
13347 %}
13348
13349 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13350 match(Set dst (SignumD src (Binary zero one)));
13351 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13352 format %{ "signumD $dst, $src" %}
13353 ins_encode %{
13354 FloatRegister src = as_FloatRegister($src$$reg),
13355 dst = as_FloatRegister($dst$$reg),
13356 zero = as_FloatRegister($zero$$reg),
13357 one = as_FloatRegister($one$$reg);
13358 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13359 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13360 // Bit selection instruction gets bit from "one" for each enabled bit in
13361 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13362 // NaN the whole "src" will be copied because "dst" is zero. For all other
13363 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13364 // from "src", and all other bits are copied from 1.0.
13365 __ bsl(dst, __ T8B, one, src);
13366 %}
13367 ins_pipe(fp_uop_d);
13368 %}
13369
13370 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13371 match(Set dst (SignumF src (Binary zero one)));
13372 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13373 format %{ "signumF $dst, $src" %}
13374 ins_encode %{
13375 FloatRegister src = as_FloatRegister($src$$reg),
13376 dst = as_FloatRegister($dst$$reg),
13377 zero = as_FloatRegister($zero$$reg),
13378 one = as_FloatRegister($one$$reg);
13379 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13380 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13381 // Bit selection instruction gets bit from "one" for each enabled bit in
13382 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13383 // NaN the whole "src" will be copied because "dst" is zero. For all other
13384 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13385 // from "src", and all other bits are copied from 1.0.
13386 __ bsl(dst, __ T8B, one, src);
13387 %}
13388 ins_pipe(fp_uop_d);
13389 %}
13390
13391 instruct onspinwait() %{
13392 match(OnSpinWait);
13393 ins_cost(INSN_COST);
13394
13395 format %{ "onspinwait" %}
13396
13397 ins_encode %{
13398 __ spin_wait();
13399 %}
13400 ins_pipe(pipe_class_empty);
13401 %}
13402
13403 // ============================================================================
13404 // Logical Instructions
13405
13406 // Integer Logical Instructions
13407
13408 // And Instructions
13409
13410
13411 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13412 match(Set dst (AndI src1 src2));
13413
13414 format %{ "andw $dst, $src1, $src2\t# int" %}
13415
13416 ins_cost(INSN_COST);
13417 ins_encode %{
13418 __ andw(as_Register($dst$$reg),
13419 as_Register($src1$$reg),
13420 as_Register($src2$$reg));
13421 %}
13422
13423 ins_pipe(ialu_reg_reg);
13424 %}
13425
13426 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13427 match(Set dst (AndI src1 src2));
13428
13429 format %{ "andsw $dst, $src1, $src2\t# int" %}
13430
13431 ins_cost(INSN_COST);
13432 ins_encode %{
13433 __ andw(as_Register($dst$$reg),
13434 as_Register($src1$$reg),
13435 (uint64_t)($src2$$constant));
13436 %}
13437
13438 ins_pipe(ialu_reg_imm);
13439 %}
13440
13441 // Or Instructions
13442
13443 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13444 match(Set dst (OrI src1 src2));
13445
13446 format %{ "orrw $dst, $src1, $src2\t# int" %}
13447
13448 ins_cost(INSN_COST);
13449 ins_encode %{
13450 __ orrw(as_Register($dst$$reg),
13451 as_Register($src1$$reg),
13452 as_Register($src2$$reg));
13453 %}
13454
13455 ins_pipe(ialu_reg_reg);
13456 %}
13457
13458 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13459 match(Set dst (OrI src1 src2));
13460
13461 format %{ "orrw $dst, $src1, $src2\t# int" %}
13462
13463 ins_cost(INSN_COST);
13464 ins_encode %{
13465 __ orrw(as_Register($dst$$reg),
13466 as_Register($src1$$reg),
13467 (uint64_t)($src2$$constant));
13468 %}
13469
13470 ins_pipe(ialu_reg_imm);
13471 %}
13472
13473 // Xor Instructions
13474
13475 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13476 match(Set dst (XorI src1 src2));
13477
13478 format %{ "eorw $dst, $src1, $src2\t# int" %}
13479
13480 ins_cost(INSN_COST);
13481 ins_encode %{
13482 __ eorw(as_Register($dst$$reg),
13483 as_Register($src1$$reg),
13484 as_Register($src2$$reg));
13485 %}
13486
13487 ins_pipe(ialu_reg_reg);
13488 %}
13489
13490 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13491 match(Set dst (XorI src1 src2));
13492
13493 format %{ "eorw $dst, $src1, $src2\t# int" %}
13494
13495 ins_cost(INSN_COST);
13496 ins_encode %{
13497 __ eorw(as_Register($dst$$reg),
13498 as_Register($src1$$reg),
13499 (uint64_t)($src2$$constant));
13500 %}
13501
13502 ins_pipe(ialu_reg_imm);
13503 %}
13504
13505 // Long Logical Instructions
13506 // TODO
13507
13508 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13509 match(Set dst (AndL src1 src2));
13510
13511 format %{ "and $dst, $src1, $src2\t# int" %}
13512
13513 ins_cost(INSN_COST);
13514 ins_encode %{
13515 __ andr(as_Register($dst$$reg),
13516 as_Register($src1$$reg),
13517 as_Register($src2$$reg));
13518 %}
13519
13520 ins_pipe(ialu_reg_reg);
13521 %}
13522
13523 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13524 match(Set dst (AndL src1 src2));
13525
13526 format %{ "and $dst, $src1, $src2\t# int" %}
13527
13528 ins_cost(INSN_COST);
13529 ins_encode %{
13530 __ andr(as_Register($dst$$reg),
13531 as_Register($src1$$reg),
13532 (uint64_t)($src2$$constant));
13533 %}
13534
13535 ins_pipe(ialu_reg_imm);
13536 %}
13537
13538 // Or Instructions
13539
13540 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13541 match(Set dst (OrL src1 src2));
13542
13543 format %{ "orr $dst, $src1, $src2\t# int" %}
13544
13545 ins_cost(INSN_COST);
13546 ins_encode %{
13547 __ orr(as_Register($dst$$reg),
13548 as_Register($src1$$reg),
13549 as_Register($src2$$reg));
13550 %}
13551
13552 ins_pipe(ialu_reg_reg);
13553 %}
13554
13555 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13556 match(Set dst (OrL src1 src2));
13557
13558 format %{ "orr $dst, $src1, $src2\t# int" %}
13559
13560 ins_cost(INSN_COST);
13561 ins_encode %{
13562 __ orr(as_Register($dst$$reg),
13563 as_Register($src1$$reg),
13564 (uint64_t)($src2$$constant));
13565 %}
13566
13567 ins_pipe(ialu_reg_imm);
13568 %}
13569
13570 // Xor Instructions
13571
13572 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13573 match(Set dst (XorL src1 src2));
13574
13575 format %{ "eor $dst, $src1, $src2\t# int" %}
13576
13577 ins_cost(INSN_COST);
13578 ins_encode %{
13579 __ eor(as_Register($dst$$reg),
13580 as_Register($src1$$reg),
13581 as_Register($src2$$reg));
13582 %}
13583
13584 ins_pipe(ialu_reg_reg);
13585 %}
13586
13587 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13588 match(Set dst (XorL src1 src2));
13589
13590 ins_cost(INSN_COST);
13591 format %{ "eor $dst, $src1, $src2\t# int" %}
13592
13593 ins_encode %{
13594 __ eor(as_Register($dst$$reg),
13595 as_Register($src1$$reg),
13596 (uint64_t)($src2$$constant));
13597 %}
13598
13599 ins_pipe(ialu_reg_imm);
13600 %}
13601
13602 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13603 %{
13604 match(Set dst (ConvI2L src));
13605
13606 ins_cost(INSN_COST);
13607 format %{ "sxtw $dst, $src\t# i2l" %}
13608 ins_encode %{
13609 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13610 %}
13611 ins_pipe(ialu_reg_shift);
13612 %}
13613
13614 // this pattern occurs in bigmath arithmetic
13615 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13616 %{
13617 match(Set dst (AndL (ConvI2L src) mask));
13618
13619 ins_cost(INSN_COST);
13620 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13621 ins_encode %{
13622 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13623 %}
13624
13625 ins_pipe(ialu_reg_shift);
13626 %}
13627
13628 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13629 match(Set dst (ConvL2I src));
13630
13631 ins_cost(INSN_COST);
13632 format %{ "movw $dst, $src \t// l2i" %}
13633
13634 ins_encode %{
13635 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13636 %}
13637
13638 ins_pipe(ialu_reg);
13639 %}
13640
13641 instruct convD2F_reg(vRegF dst, vRegD src) %{
13642 match(Set dst (ConvD2F src));
13643
13644 ins_cost(INSN_COST * 5);
13645 format %{ "fcvtd $dst, $src \t// d2f" %}
13646
13647 ins_encode %{
13648 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13649 %}
13650
13651 ins_pipe(fp_d2f);
13652 %}
13653
13654 instruct convF2D_reg(vRegD dst, vRegF src) %{
13655 match(Set dst (ConvF2D src));
13656
13657 ins_cost(INSN_COST * 5);
13658 format %{ "fcvts $dst, $src \t// f2d" %}
13659
13660 ins_encode %{
13661 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13662 %}
13663
13664 ins_pipe(fp_f2d);
13665 %}
13666
13667 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13668 match(Set dst (ConvF2I src));
13669
13670 ins_cost(INSN_COST * 5);
13671 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13672
13673 ins_encode %{
13674 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13675 %}
13676
13677 ins_pipe(fp_f2i);
13678 %}
13679
13680 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13681 match(Set dst (ConvF2L src));
13682
13683 ins_cost(INSN_COST * 5);
13684 format %{ "fcvtzs $dst, $src \t// f2l" %}
13685
13686 ins_encode %{
13687 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13688 %}
13689
13690 ins_pipe(fp_f2l);
13691 %}
13692
13693 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13694 match(Set dst (ConvF2HF src));
13695 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13696 "smov $dst, $tmp\t# move result from $tmp to $dst"
13697 %}
13698 effect(TEMP tmp);
13699 ins_encode %{
13700 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13701 %}
13702 ins_pipe(pipe_slow);
13703 %}
13704
13705 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13706 match(Set dst (ConvHF2F src));
13707 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13708 "fcvt $dst, $tmp\t# convert half to single precision"
13709 %}
13710 effect(TEMP tmp);
13711 ins_encode %{
13712 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13713 %}
13714 ins_pipe(pipe_slow);
13715 %}
13716
13717 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13718 match(Set dst (ConvI2F src));
13719
13720 ins_cost(INSN_COST * 5);
13721 format %{ "scvtfws $dst, $src \t// i2f" %}
13722
13723 ins_encode %{
13724 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13725 %}
13726
13727 ins_pipe(fp_i2f);
13728 %}
13729
13730 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13731 match(Set dst (ConvL2F src));
13732
13733 ins_cost(INSN_COST * 5);
13734 format %{ "scvtfs $dst, $src \t// l2f" %}
13735
13736 ins_encode %{
13737 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13738 %}
13739
13740 ins_pipe(fp_l2f);
13741 %}
13742
13743 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13744 match(Set dst (ConvD2I src));
13745
13746 ins_cost(INSN_COST * 5);
13747 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13748
13749 ins_encode %{
13750 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13751 %}
13752
13753 ins_pipe(fp_d2i);
13754 %}
13755
13756 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13757 match(Set dst (ConvD2L src));
13758
13759 ins_cost(INSN_COST * 5);
13760 format %{ "fcvtzd $dst, $src \t// d2l" %}
13761
13762 ins_encode %{
13763 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13764 %}
13765
13766 ins_pipe(fp_d2l);
13767 %}
13768
13769 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13770 match(Set dst (ConvI2D src));
13771
13772 ins_cost(INSN_COST * 5);
13773 format %{ "scvtfwd $dst, $src \t// i2d" %}
13774
13775 ins_encode %{
13776 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13777 %}
13778
13779 ins_pipe(fp_i2d);
13780 %}
13781
13782 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13783 match(Set dst (ConvL2D src));
13784
13785 ins_cost(INSN_COST * 5);
13786 format %{ "scvtfd $dst, $src \t// l2d" %}
13787
13788 ins_encode %{
13789 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13790 %}
13791
13792 ins_pipe(fp_l2d);
13793 %}
13794
13795 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13796 %{
13797 match(Set dst (RoundD src));
13798 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13799 format %{ "java_round_double $dst,$src"%}
13800 ins_encode %{
13801 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13802 as_FloatRegister($ftmp$$reg));
13803 %}
13804 ins_pipe(pipe_slow);
13805 %}
13806
13807 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13808 %{
13809 match(Set dst (RoundF src));
13810 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13811 format %{ "java_round_float $dst,$src"%}
13812 ins_encode %{
13813 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13814 as_FloatRegister($ftmp$$reg));
13815 %}
13816 ins_pipe(pipe_slow);
13817 %}
13818
13819 // stack <-> reg and reg <-> reg shuffles with no conversion
13820
13821 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13822
13823 match(Set dst (MoveF2I src));
13824
13825 effect(DEF dst, USE src);
13826
13827 ins_cost(4 * INSN_COST);
13828
13829 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13830
13831 ins_encode %{
13832 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13833 %}
13834
13835 ins_pipe(iload_reg_reg);
13836
13837 %}
13838
13839 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13840
13841 match(Set dst (MoveI2F src));
13842
13843 effect(DEF dst, USE src);
13844
13845 ins_cost(4 * INSN_COST);
13846
13847 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13848
13849 ins_encode %{
13850 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13851 %}
13852
13853 ins_pipe(pipe_class_memory);
13854
13855 %}
13856
13857 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13858
13859 match(Set dst (MoveD2L src));
13860
13861 effect(DEF dst, USE src);
13862
13863 ins_cost(4 * INSN_COST);
13864
13865 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13866
13867 ins_encode %{
13868 __ ldr($dst$$Register, Address(sp, $src$$disp));
13869 %}
13870
13871 ins_pipe(iload_reg_reg);
13872
13873 %}
13874
13875 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13876
13877 match(Set dst (MoveL2D src));
13878
13879 effect(DEF dst, USE src);
13880
13881 ins_cost(4 * INSN_COST);
13882
13883 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13884
13885 ins_encode %{
13886 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13887 %}
13888
13889 ins_pipe(pipe_class_memory);
13890
13891 %}
13892
13893 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13894
13895 match(Set dst (MoveF2I src));
13896
13897 effect(DEF dst, USE src);
13898
13899 ins_cost(INSN_COST);
13900
13901 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13902
13903 ins_encode %{
13904 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13905 %}
13906
13907 ins_pipe(pipe_class_memory);
13908
13909 %}
13910
13911 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13912
13913 match(Set dst (MoveI2F src));
13914
13915 effect(DEF dst, USE src);
13916
13917 ins_cost(INSN_COST);
13918
13919 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13920
13921 ins_encode %{
13922 __ strw($src$$Register, Address(sp, $dst$$disp));
13923 %}
13924
13925 ins_pipe(istore_reg_reg);
13926
13927 %}
13928
13929 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13930
13931 match(Set dst (MoveD2L src));
13932
13933 effect(DEF dst, USE src);
13934
13935 ins_cost(INSN_COST);
13936
13937 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13938
13939 ins_encode %{
13940 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13941 %}
13942
13943 ins_pipe(pipe_class_memory);
13944
13945 %}
13946
13947 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13948
13949 match(Set dst (MoveL2D src));
13950
13951 effect(DEF dst, USE src);
13952
13953 ins_cost(INSN_COST);
13954
13955 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13956
13957 ins_encode %{
13958 __ str($src$$Register, Address(sp, $dst$$disp));
13959 %}
13960
13961 ins_pipe(istore_reg_reg);
13962
13963 %}
13964
13965 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13966
13967 match(Set dst (MoveF2I src));
13968
13969 effect(DEF dst, USE src);
13970
13971 ins_cost(INSN_COST);
13972
13973 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
13974
13975 ins_encode %{
13976 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
13977 %}
13978
13979 ins_pipe(fp_f2i);
13980
13981 %}
13982
13983 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
13984
13985 match(Set dst (MoveI2F src));
13986
13987 effect(DEF dst, USE src);
13988
13989 ins_cost(INSN_COST);
13990
13991 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
13992
13993 ins_encode %{
13994 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
13995 %}
13996
13997 ins_pipe(fp_i2f);
13998
13999 %}
14000
14001 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14002
14003 match(Set dst (MoveD2L src));
14004
14005 effect(DEF dst, USE src);
14006
14007 ins_cost(INSN_COST);
14008
14009 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14010
14011 ins_encode %{
14012 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14013 %}
14014
14015 ins_pipe(fp_d2l);
14016
14017 %}
14018
14019 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14020
14021 match(Set dst (MoveL2D src));
14022
14023 effect(DEF dst, USE src);
14024
14025 ins_cost(INSN_COST);
14026
14027 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14028
14029 ins_encode %{
14030 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14031 %}
14032
14033 ins_pipe(fp_l2d);
14034
14035 %}
14036
14037 // ============================================================================
14038 // clearing of an array
14039
14040 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14041 %{
14042 match(Set dummy (ClearArray cnt base));
14043 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14044
14045 ins_cost(4 * INSN_COST);
14046 format %{ "ClearArray $cnt, $base" %}
14047
14048 ins_encode %{
14049 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14050 if (tpc == nullptr) {
14051 ciEnv::current()->record_failure("CodeCache is full");
14052 return;
14053 }
14054 %}
14055
14056 ins_pipe(pipe_class_memory);
14057 %}
14058
14059 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14060 %{
14061 predicate((uint64_t)n->in(2)->get_long()
14062 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14063 match(Set dummy (ClearArray cnt base));
14064 effect(TEMP temp, USE_KILL base, KILL cr);
14065
14066 ins_cost(4 * INSN_COST);
14067 format %{ "ClearArray $cnt, $base" %}
14068
14069 ins_encode %{
14070 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14071 if (tpc == nullptr) {
14072 ciEnv::current()->record_failure("CodeCache is full");
14073 return;
14074 }
14075 %}
14076
14077 ins_pipe(pipe_class_memory);
14078 %}
14079
14080 // ============================================================================
14081 // Overflow Math Instructions
14082
14083 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14084 %{
14085 match(Set cr (OverflowAddI op1 op2));
14086
14087 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14088 ins_cost(INSN_COST);
14089 ins_encode %{
14090 __ cmnw($op1$$Register, $op2$$Register);
14091 %}
14092
14093 ins_pipe(icmp_reg_reg);
14094 %}
14095
14096 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14097 %{
14098 match(Set cr (OverflowAddI op1 op2));
14099
14100 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14101 ins_cost(INSN_COST);
14102 ins_encode %{
14103 __ cmnw($op1$$Register, $op2$$constant);
14104 %}
14105
14106 ins_pipe(icmp_reg_imm);
14107 %}
14108
14109 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14110 %{
14111 match(Set cr (OverflowAddL op1 op2));
14112
14113 format %{ "cmn $op1, $op2\t# overflow check long" %}
14114 ins_cost(INSN_COST);
14115 ins_encode %{
14116 __ cmn($op1$$Register, $op2$$Register);
14117 %}
14118
14119 ins_pipe(icmp_reg_reg);
14120 %}
14121
14122 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14123 %{
14124 match(Set cr (OverflowAddL op1 op2));
14125
14126 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14127 ins_cost(INSN_COST);
14128 ins_encode %{
14129 __ adds(zr, $op1$$Register, $op2$$constant);
14130 %}
14131
14132 ins_pipe(icmp_reg_imm);
14133 %}
14134
14135 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14136 %{
14137 match(Set cr (OverflowSubI op1 op2));
14138
14139 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14140 ins_cost(INSN_COST);
14141 ins_encode %{
14142 __ cmpw($op1$$Register, $op2$$Register);
14143 %}
14144
14145 ins_pipe(icmp_reg_reg);
14146 %}
14147
14148 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14149 %{
14150 match(Set cr (OverflowSubI op1 op2));
14151
14152 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14153 ins_cost(INSN_COST);
14154 ins_encode %{
14155 __ cmpw($op1$$Register, $op2$$constant);
14156 %}
14157
14158 ins_pipe(icmp_reg_imm);
14159 %}
14160
14161 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14162 %{
14163 match(Set cr (OverflowSubL op1 op2));
14164
14165 format %{ "cmp $op1, $op2\t# overflow check long" %}
14166 ins_cost(INSN_COST);
14167 ins_encode %{
14168 __ cmp($op1$$Register, $op2$$Register);
14169 %}
14170
14171 ins_pipe(icmp_reg_reg);
14172 %}
14173
14174 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14175 %{
14176 match(Set cr (OverflowSubL op1 op2));
14177
14178 format %{ "cmp $op1, $op2\t# overflow check long" %}
14179 ins_cost(INSN_COST);
14180 ins_encode %{
14181 __ subs(zr, $op1$$Register, $op2$$constant);
14182 %}
14183
14184 ins_pipe(icmp_reg_imm);
14185 %}
14186
14187 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14188 %{
14189 match(Set cr (OverflowSubI zero op1));
14190
14191 format %{ "cmpw zr, $op1\t# overflow check int" %}
14192 ins_cost(INSN_COST);
14193 ins_encode %{
14194 __ cmpw(zr, $op1$$Register);
14195 %}
14196
14197 ins_pipe(icmp_reg_imm);
14198 %}
14199
14200 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14201 %{
14202 match(Set cr (OverflowSubL zero op1));
14203
14204 format %{ "cmp zr, $op1\t# overflow check long" %}
14205 ins_cost(INSN_COST);
14206 ins_encode %{
14207 __ cmp(zr, $op1$$Register);
14208 %}
14209
14210 ins_pipe(icmp_reg_imm);
14211 %}
14212
14213 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14214 %{
14215 match(Set cr (OverflowMulI op1 op2));
14216
14217 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14218 "cmp rscratch1, rscratch1, sxtw\n\t"
14219 "movw rscratch1, #0x80000000\n\t"
14220 "cselw rscratch1, rscratch1, zr, NE\n\t"
14221 "cmpw rscratch1, #1" %}
14222 ins_cost(5 * INSN_COST);
14223 ins_encode %{
14224 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14225 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14226 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14227 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14228 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14229 %}
14230
14231 ins_pipe(pipe_slow);
14232 %}
14233
14234 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14235 %{
14236 match(If cmp (OverflowMulI op1 op2));
14237 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14238 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14239 effect(USE labl, KILL cr);
14240
14241 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14242 "cmp rscratch1, rscratch1, sxtw\n\t"
14243 "b$cmp $labl" %}
14244 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14245 ins_encode %{
14246 Label* L = $labl$$label;
14247 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14248 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14249 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14250 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14251 %}
14252
14253 ins_pipe(pipe_serial);
14254 %}
14255
14256 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14257 %{
14258 match(Set cr (OverflowMulL op1 op2));
14259
14260 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14261 "smulh rscratch2, $op1, $op2\n\t"
14262 "cmp rscratch2, rscratch1, ASR #63\n\t"
14263 "movw rscratch1, #0x80000000\n\t"
14264 "cselw rscratch1, rscratch1, zr, NE\n\t"
14265 "cmpw rscratch1, #1" %}
14266 ins_cost(6 * INSN_COST);
14267 ins_encode %{
14268 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14269 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14270 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14271 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14272 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14273 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14274 %}
14275
14276 ins_pipe(pipe_slow);
14277 %}
14278
14279 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14280 %{
14281 match(If cmp (OverflowMulL op1 op2));
14282 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14283 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14284 effect(USE labl, KILL cr);
14285
14286 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14287 "smulh rscratch2, $op1, $op2\n\t"
14288 "cmp rscratch2, rscratch1, ASR #63\n\t"
14289 "b$cmp $labl" %}
14290 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14291 ins_encode %{
14292 Label* L = $labl$$label;
14293 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14294 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14295 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14296 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14297 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14298 %}
14299
14300 ins_pipe(pipe_serial);
14301 %}
14302
14303 // ============================================================================
14304 // Compare Instructions
14305
14306 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14307 %{
14308 match(Set cr (CmpI op1 op2));
14309
14310 effect(DEF cr, USE op1, USE op2);
14311
14312 ins_cost(INSN_COST);
14313 format %{ "cmpw $op1, $op2" %}
14314
14315 ins_encode(aarch64_enc_cmpw(op1, op2));
14316
14317 ins_pipe(icmp_reg_reg);
14318 %}
14319
14320 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14321 %{
14322 match(Set cr (CmpI op1 zero));
14323
14324 effect(DEF cr, USE op1);
14325
14326 ins_cost(INSN_COST);
14327 format %{ "cmpw $op1, 0" %}
14328
14329 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14330
14331 ins_pipe(icmp_reg_imm);
14332 %}
14333
14334 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14335 %{
14336 match(Set cr (CmpI op1 op2));
14337
14338 effect(DEF cr, USE op1);
14339
14340 ins_cost(INSN_COST);
14341 format %{ "cmpw $op1, $op2" %}
14342
14343 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14344
14345 ins_pipe(icmp_reg_imm);
14346 %}
14347
14348 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14349 %{
14350 match(Set cr (CmpI op1 op2));
14351
14352 effect(DEF cr, USE op1);
14353
14354 ins_cost(INSN_COST * 2);
14355 format %{ "cmpw $op1, $op2" %}
14356
14357 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14358
14359 ins_pipe(icmp_reg_imm);
14360 %}
14361
14362 // Unsigned compare Instructions; really, same as signed compare
14363 // except it should only be used to feed an If or a CMovI which takes a
14364 // cmpOpU.
14365
14366 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14367 %{
14368 match(Set cr (CmpU op1 op2));
14369
14370 effect(DEF cr, USE op1, USE op2);
14371
14372 ins_cost(INSN_COST);
14373 format %{ "cmpw $op1, $op2\t# unsigned" %}
14374
14375 ins_encode(aarch64_enc_cmpw(op1, op2));
14376
14377 ins_pipe(icmp_reg_reg);
14378 %}
14379
14380 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14381 %{
14382 match(Set cr (CmpU op1 zero));
14383
14384 effect(DEF cr, USE op1);
14385
14386 ins_cost(INSN_COST);
14387 format %{ "cmpw $op1, #0\t# unsigned" %}
14388
14389 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14390
14391 ins_pipe(icmp_reg_imm);
14392 %}
14393
14394 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14395 %{
14396 match(Set cr (CmpU op1 op2));
14397
14398 effect(DEF cr, USE op1);
14399
14400 ins_cost(INSN_COST);
14401 format %{ "cmpw $op1, $op2\t# unsigned" %}
14402
14403 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14404
14405 ins_pipe(icmp_reg_imm);
14406 %}
14407
14408 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14409 %{
14410 match(Set cr (CmpU op1 op2));
14411
14412 effect(DEF cr, USE op1);
14413
14414 ins_cost(INSN_COST * 2);
14415 format %{ "cmpw $op1, $op2\t# unsigned" %}
14416
14417 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14418
14419 ins_pipe(icmp_reg_imm);
14420 %}
14421
14422 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14423 %{
14424 match(Set cr (CmpL op1 op2));
14425
14426 effect(DEF cr, USE op1, USE op2);
14427
14428 ins_cost(INSN_COST);
14429 format %{ "cmp $op1, $op2" %}
14430
14431 ins_encode(aarch64_enc_cmp(op1, op2));
14432
14433 ins_pipe(icmp_reg_reg);
14434 %}
14435
14436 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14437 %{
14438 match(Set cr (CmpL op1 zero));
14439
14440 effect(DEF cr, USE op1);
14441
14442 ins_cost(INSN_COST);
14443 format %{ "tst $op1" %}
14444
14445 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14446
14447 ins_pipe(icmp_reg_imm);
14448 %}
14449
14450 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14451 %{
14452 match(Set cr (CmpL op1 op2));
14453
14454 effect(DEF cr, USE op1);
14455
14456 ins_cost(INSN_COST);
14457 format %{ "cmp $op1, $op2" %}
14458
14459 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14460
14461 ins_pipe(icmp_reg_imm);
14462 %}
14463
14464 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14465 %{
14466 match(Set cr (CmpL op1 op2));
14467
14468 effect(DEF cr, USE op1);
14469
14470 ins_cost(INSN_COST * 2);
14471 format %{ "cmp $op1, $op2" %}
14472
14473 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14474
14475 ins_pipe(icmp_reg_imm);
14476 %}
14477
14478 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14479 %{
14480 match(Set cr (CmpUL op1 op2));
14481
14482 effect(DEF cr, USE op1, USE op2);
14483
14484 ins_cost(INSN_COST);
14485 format %{ "cmp $op1, $op2" %}
14486
14487 ins_encode(aarch64_enc_cmp(op1, op2));
14488
14489 ins_pipe(icmp_reg_reg);
14490 %}
14491
14492 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14493 %{
14494 match(Set cr (CmpUL op1 zero));
14495
14496 effect(DEF cr, USE op1);
14497
14498 ins_cost(INSN_COST);
14499 format %{ "tst $op1" %}
14500
14501 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14502
14503 ins_pipe(icmp_reg_imm);
14504 %}
14505
14506 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14507 %{
14508 match(Set cr (CmpUL op1 op2));
14509
14510 effect(DEF cr, USE op1);
14511
14512 ins_cost(INSN_COST);
14513 format %{ "cmp $op1, $op2" %}
14514
14515 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14516
14517 ins_pipe(icmp_reg_imm);
14518 %}
14519
14520 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14521 %{
14522 match(Set cr (CmpUL op1 op2));
14523
14524 effect(DEF cr, USE op1);
14525
14526 ins_cost(INSN_COST * 2);
14527 format %{ "cmp $op1, $op2" %}
14528
14529 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14530
14531 ins_pipe(icmp_reg_imm);
14532 %}
14533
14534 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14535 %{
14536 match(Set cr (CmpP op1 op2));
14537
14538 effect(DEF cr, USE op1, USE op2);
14539
14540 ins_cost(INSN_COST);
14541 format %{ "cmp $op1, $op2\t // ptr" %}
14542
14543 ins_encode(aarch64_enc_cmpp(op1, op2));
14544
14545 ins_pipe(icmp_reg_reg);
14546 %}
14547
14548 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14549 %{
14550 match(Set cr (CmpN op1 op2));
14551
14552 effect(DEF cr, USE op1, USE op2);
14553
14554 ins_cost(INSN_COST);
14555 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14556
14557 ins_encode(aarch64_enc_cmpn(op1, op2));
14558
14559 ins_pipe(icmp_reg_reg);
14560 %}
14561
14562 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14563 %{
14564 match(Set cr (CmpP op1 zero));
14565
14566 effect(DEF cr, USE op1, USE zero);
14567
14568 ins_cost(INSN_COST);
14569 format %{ "cmp $op1, 0\t // ptr" %}
14570
14571 ins_encode(aarch64_enc_testp(op1));
14572
14573 ins_pipe(icmp_reg_imm);
14574 %}
14575
14576 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14577 %{
14578 match(Set cr (CmpN op1 zero));
14579
14580 effect(DEF cr, USE op1, USE zero);
14581
14582 ins_cost(INSN_COST);
14583 format %{ "cmp $op1, 0\t // compressed ptr" %}
14584
14585 ins_encode(aarch64_enc_testn(op1));
14586
14587 ins_pipe(icmp_reg_imm);
14588 %}
14589
14590 // FP comparisons
14591 //
14592 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14593 // using normal cmpOp. See declaration of rFlagsReg for details.
14594
14595 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14596 %{
14597 match(Set cr (CmpF src1 src2));
14598
14599 ins_cost(3 * INSN_COST);
14600 format %{ "fcmps $src1, $src2" %}
14601
14602 ins_encode %{
14603 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14604 %}
14605
14606 ins_pipe(pipe_class_compare);
14607 %}
14608
14609 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14610 %{
14611 match(Set cr (CmpF src1 src2));
14612
14613 ins_cost(3 * INSN_COST);
14614 format %{ "fcmps $src1, 0.0" %}
14615
14616 ins_encode %{
14617 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14618 %}
14619
14620 ins_pipe(pipe_class_compare);
14621 %}
14622 // FROM HERE
14623
14624 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14625 %{
14626 match(Set cr (CmpD src1 src2));
14627
14628 ins_cost(3 * INSN_COST);
14629 format %{ "fcmpd $src1, $src2" %}
14630
14631 ins_encode %{
14632 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14633 %}
14634
14635 ins_pipe(pipe_class_compare);
14636 %}
14637
14638 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14639 %{
14640 match(Set cr (CmpD src1 src2));
14641
14642 ins_cost(3 * INSN_COST);
14643 format %{ "fcmpd $src1, 0.0" %}
14644
14645 ins_encode %{
14646 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14647 %}
14648
14649 ins_pipe(pipe_class_compare);
14650 %}
14651
14652 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14653 %{
14654 match(Set dst (CmpF3 src1 src2));
14655 effect(KILL cr);
14656
14657 ins_cost(5 * INSN_COST);
14658 format %{ "fcmps $src1, $src2\n\t"
14659 "csinvw($dst, zr, zr, eq\n\t"
14660 "csnegw($dst, $dst, $dst, lt)"
14661 %}
14662
14663 ins_encode %{
14664 Label done;
14665 FloatRegister s1 = as_FloatRegister($src1$$reg);
14666 FloatRegister s2 = as_FloatRegister($src2$$reg);
14667 Register d = as_Register($dst$$reg);
14668 __ fcmps(s1, s2);
14669 // installs 0 if EQ else -1
14670 __ csinvw(d, zr, zr, Assembler::EQ);
14671 // keeps -1 if less or unordered else installs 1
14672 __ csnegw(d, d, d, Assembler::LT);
14673 __ bind(done);
14674 %}
14675
14676 ins_pipe(pipe_class_default);
14677
14678 %}
14679
14680 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14681 %{
14682 match(Set dst (CmpD3 src1 src2));
14683 effect(KILL cr);
14684
14685 ins_cost(5 * INSN_COST);
14686 format %{ "fcmpd $src1, $src2\n\t"
14687 "csinvw($dst, zr, zr, eq\n\t"
14688 "csnegw($dst, $dst, $dst, lt)"
14689 %}
14690
14691 ins_encode %{
14692 Label done;
14693 FloatRegister s1 = as_FloatRegister($src1$$reg);
14694 FloatRegister s2 = as_FloatRegister($src2$$reg);
14695 Register d = as_Register($dst$$reg);
14696 __ fcmpd(s1, s2);
14697 // installs 0 if EQ else -1
14698 __ csinvw(d, zr, zr, Assembler::EQ);
14699 // keeps -1 if less or unordered else installs 1
14700 __ csnegw(d, d, d, Assembler::LT);
14701 __ bind(done);
14702 %}
14703 ins_pipe(pipe_class_default);
14704
14705 %}
14706
14707 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14708 %{
14709 match(Set dst (CmpF3 src1 zero));
14710 effect(KILL cr);
14711
14712 ins_cost(5 * INSN_COST);
14713 format %{ "fcmps $src1, 0.0\n\t"
14714 "csinvw($dst, zr, zr, eq\n\t"
14715 "csnegw($dst, $dst, $dst, lt)"
14716 %}
14717
14718 ins_encode %{
14719 Label done;
14720 FloatRegister s1 = as_FloatRegister($src1$$reg);
14721 Register d = as_Register($dst$$reg);
14722 __ fcmps(s1, 0.0);
14723 // installs 0 if EQ else -1
14724 __ csinvw(d, zr, zr, Assembler::EQ);
14725 // keeps -1 if less or unordered else installs 1
14726 __ csnegw(d, d, d, Assembler::LT);
14727 __ bind(done);
14728 %}
14729
14730 ins_pipe(pipe_class_default);
14731
14732 %}
14733
14734 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14735 %{
14736 match(Set dst (CmpD3 src1 zero));
14737 effect(KILL cr);
14738
14739 ins_cost(5 * INSN_COST);
14740 format %{ "fcmpd $src1, 0.0\n\t"
14741 "csinvw($dst, zr, zr, eq\n\t"
14742 "csnegw($dst, $dst, $dst, lt)"
14743 %}
14744
14745 ins_encode %{
14746 Label done;
14747 FloatRegister s1 = as_FloatRegister($src1$$reg);
14748 Register d = as_Register($dst$$reg);
14749 __ fcmpd(s1, 0.0);
14750 // installs 0 if EQ else -1
14751 __ csinvw(d, zr, zr, Assembler::EQ);
14752 // keeps -1 if less or unordered else installs 1
14753 __ csnegw(d, d, d, Assembler::LT);
14754 __ bind(done);
14755 %}
14756 ins_pipe(pipe_class_default);
14757
14758 %}
14759
14760 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14761 %{
14762 match(Set dst (CmpLTMask p q));
14763 effect(KILL cr);
14764
14765 ins_cost(3 * INSN_COST);
14766
14767 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14768 "csetw $dst, lt\n\t"
14769 "subw $dst, zr, $dst"
14770 %}
14771
14772 ins_encode %{
14773 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14774 __ csetw(as_Register($dst$$reg), Assembler::LT);
14775 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14776 %}
14777
14778 ins_pipe(ialu_reg_reg);
14779 %}
14780
14781 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14782 %{
14783 match(Set dst (CmpLTMask src zero));
14784 effect(KILL cr);
14785
14786 ins_cost(INSN_COST);
14787
14788 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14789
14790 ins_encode %{
14791 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14792 %}
14793
14794 ins_pipe(ialu_reg_shift);
14795 %}
14796
14797 // ============================================================================
14798 // Max and Min
14799
14800 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14801
14802 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14803 %{
14804 effect(DEF cr, USE src);
14805 ins_cost(INSN_COST);
14806 format %{ "cmpw $src, 0" %}
14807
14808 ins_encode %{
14809 __ cmpw($src$$Register, 0);
14810 %}
14811 ins_pipe(icmp_reg_imm);
14812 %}
14813
14814 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14815 %{
14816 match(Set dst (MinI src1 src2));
14817 ins_cost(INSN_COST * 3);
14818
14819 expand %{
14820 rFlagsReg cr;
14821 compI_reg_reg(cr, src1, src2);
14822 cmovI_reg_reg_lt(dst, src1, src2, cr);
14823 %}
14824 %}
14825
14826 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14827 %{
14828 match(Set dst (MaxI src1 src2));
14829 ins_cost(INSN_COST * 3);
14830
14831 expand %{
14832 rFlagsReg cr;
14833 compI_reg_reg(cr, src1, src2);
14834 cmovI_reg_reg_gt(dst, src1, src2, cr);
14835 %}
14836 %}
14837
14838
14839 // ============================================================================
14840 // Branch Instructions
14841
14842 // Direct Branch.
14843 instruct branch(label lbl)
14844 %{
14845 match(Goto);
14846
14847 effect(USE lbl);
14848
14849 ins_cost(BRANCH_COST);
14850 format %{ "b $lbl" %}
14851
14852 ins_encode(aarch64_enc_b(lbl));
14853
14854 ins_pipe(pipe_branch);
14855 %}
14856
14857 // Conditional Near Branch
14858 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14859 %{
14860 // Same match rule as `branchConFar'.
14861 match(If cmp cr);
14862
14863 effect(USE lbl);
14864
14865 ins_cost(BRANCH_COST);
14866 // If set to 1 this indicates that the current instruction is a
14867 // short variant of a long branch. This avoids using this
14868 // instruction in first-pass matching. It will then only be used in
14869 // the `Shorten_branches' pass.
14870 // ins_short_branch(1);
14871 format %{ "b$cmp $lbl" %}
14872
14873 ins_encode(aarch64_enc_br_con(cmp, lbl));
14874
14875 ins_pipe(pipe_branch_cond);
14876 %}
14877
14878 // Conditional Near Branch Unsigned
14879 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14880 %{
14881 // Same match rule as `branchConFar'.
14882 match(If cmp cr);
14883
14884 effect(USE lbl);
14885
14886 ins_cost(BRANCH_COST);
14887 // If set to 1 this indicates that the current instruction is a
14888 // short variant of a long branch. This avoids using this
14889 // instruction in first-pass matching. It will then only be used in
14890 // the `Shorten_branches' pass.
14891 // ins_short_branch(1);
14892 format %{ "b$cmp $lbl\t# unsigned" %}
14893
14894 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14895
14896 ins_pipe(pipe_branch_cond);
14897 %}
14898
14899 // Make use of CBZ and CBNZ. These instructions, as well as being
14900 // shorter than (cmp; branch), have the additional benefit of not
14901 // killing the flags.
14902
14903 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14904 match(If cmp (CmpI op1 op2));
14905 effect(USE labl);
14906
14907 ins_cost(BRANCH_COST);
14908 format %{ "cbw$cmp $op1, $labl" %}
14909 ins_encode %{
14910 Label* L = $labl$$label;
14911 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14912 if (cond == Assembler::EQ)
14913 __ cbzw($op1$$Register, *L);
14914 else
14915 __ cbnzw($op1$$Register, *L);
14916 %}
14917 ins_pipe(pipe_cmp_branch);
14918 %}
14919
14920 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14921 match(If cmp (CmpL op1 op2));
14922 effect(USE labl);
14923
14924 ins_cost(BRANCH_COST);
14925 format %{ "cb$cmp $op1, $labl" %}
14926 ins_encode %{
14927 Label* L = $labl$$label;
14928 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14929 if (cond == Assembler::EQ)
14930 __ cbz($op1$$Register, *L);
14931 else
14932 __ cbnz($op1$$Register, *L);
14933 %}
14934 ins_pipe(pipe_cmp_branch);
14935 %}
14936
14937 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14938 match(If cmp (CmpP op1 op2));
14939 effect(USE labl);
14940
14941 ins_cost(BRANCH_COST);
14942 format %{ "cb$cmp $op1, $labl" %}
14943 ins_encode %{
14944 Label* L = $labl$$label;
14945 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14946 if (cond == Assembler::EQ)
14947 __ cbz($op1$$Register, *L);
14948 else
14949 __ cbnz($op1$$Register, *L);
14950 %}
14951 ins_pipe(pipe_cmp_branch);
14952 %}
14953
14954 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14955 match(If cmp (CmpN op1 op2));
14956 effect(USE labl);
14957
14958 ins_cost(BRANCH_COST);
14959 format %{ "cbw$cmp $op1, $labl" %}
14960 ins_encode %{
14961 Label* L = $labl$$label;
14962 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14963 if (cond == Assembler::EQ)
14964 __ cbzw($op1$$Register, *L);
14965 else
14966 __ cbnzw($op1$$Register, *L);
14967 %}
14968 ins_pipe(pipe_cmp_branch);
14969 %}
14970
14971 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
14972 match(If cmp (CmpP (DecodeN oop) zero));
14973 effect(USE labl);
14974
14975 ins_cost(BRANCH_COST);
14976 format %{ "cb$cmp $oop, $labl" %}
14977 ins_encode %{
14978 Label* L = $labl$$label;
14979 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14980 if (cond == Assembler::EQ)
14981 __ cbzw($oop$$Register, *L);
14982 else
14983 __ cbnzw($oop$$Register, *L);
14984 %}
14985 ins_pipe(pipe_cmp_branch);
14986 %}
14987
14988 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
14989 match(If cmp (CmpU op1 op2));
14990 effect(USE labl);
14991
14992 ins_cost(BRANCH_COST);
14993 format %{ "cbw$cmp $op1, $labl" %}
14994 ins_encode %{
14995 Label* L = $labl$$label;
14996 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14997 if (cond == Assembler::EQ || cond == Assembler::LS) {
14998 __ cbzw($op1$$Register, *L);
14999 } else {
15000 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15001 __ cbnzw($op1$$Register, *L);
15002 }
15003 %}
15004 ins_pipe(pipe_cmp_branch);
15005 %}
15006
15007 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15008 match(If cmp (CmpUL op1 op2));
15009 effect(USE labl);
15010
15011 ins_cost(BRANCH_COST);
15012 format %{ "cb$cmp $op1, $labl" %}
15013 ins_encode %{
15014 Label* L = $labl$$label;
15015 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15016 if (cond == Assembler::EQ || cond == Assembler::LS) {
15017 __ cbz($op1$$Register, *L);
15018 } else {
15019 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15020 __ cbnz($op1$$Register, *L);
15021 }
15022 %}
15023 ins_pipe(pipe_cmp_branch);
15024 %}
15025
15026 // Test bit and Branch
15027
15028 // Patterns for short (< 32KiB) variants
15029 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15030 match(If cmp (CmpL op1 op2));
15031 effect(USE labl);
15032
15033 ins_cost(BRANCH_COST);
15034 format %{ "cb$cmp $op1, $labl # long" %}
15035 ins_encode %{
15036 Label* L = $labl$$label;
15037 Assembler::Condition cond =
15038 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15039 __ tbr(cond, $op1$$Register, 63, *L);
15040 %}
15041 ins_pipe(pipe_cmp_branch);
15042 ins_short_branch(1);
15043 %}
15044
15045 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15046 match(If cmp (CmpI op1 op2));
15047 effect(USE labl);
15048
15049 ins_cost(BRANCH_COST);
15050 format %{ "cb$cmp $op1, $labl # int" %}
15051 ins_encode %{
15052 Label* L = $labl$$label;
15053 Assembler::Condition cond =
15054 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15055 __ tbr(cond, $op1$$Register, 31, *L);
15056 %}
15057 ins_pipe(pipe_cmp_branch);
15058 ins_short_branch(1);
15059 %}
15060
15061 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15062 match(If cmp (CmpL (AndL op1 op2) op3));
15063 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15064 effect(USE labl);
15065
15066 ins_cost(BRANCH_COST);
15067 format %{ "tb$cmp $op1, $op2, $labl" %}
15068 ins_encode %{
15069 Label* L = $labl$$label;
15070 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15071 int bit = exact_log2_long($op2$$constant);
15072 __ tbr(cond, $op1$$Register, bit, *L);
15073 %}
15074 ins_pipe(pipe_cmp_branch);
15075 ins_short_branch(1);
15076 %}
15077
15078 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15079 match(If cmp (CmpI (AndI op1 op2) op3));
15080 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15081 effect(USE labl);
15082
15083 ins_cost(BRANCH_COST);
15084 format %{ "tb$cmp $op1, $op2, $labl" %}
15085 ins_encode %{
15086 Label* L = $labl$$label;
15087 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15088 int bit = exact_log2((juint)$op2$$constant);
15089 __ tbr(cond, $op1$$Register, bit, *L);
15090 %}
15091 ins_pipe(pipe_cmp_branch);
15092 ins_short_branch(1);
15093 %}
15094
15095 // And far variants
15096 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15097 match(If cmp (CmpL op1 op2));
15098 effect(USE labl);
15099
15100 ins_cost(BRANCH_COST);
15101 format %{ "cb$cmp $op1, $labl # long" %}
15102 ins_encode %{
15103 Label* L = $labl$$label;
15104 Assembler::Condition cond =
15105 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15106 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15107 %}
15108 ins_pipe(pipe_cmp_branch);
15109 %}
15110
15111 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15112 match(If cmp (CmpI op1 op2));
15113 effect(USE labl);
15114
15115 ins_cost(BRANCH_COST);
15116 format %{ "cb$cmp $op1, $labl # int" %}
15117 ins_encode %{
15118 Label* L = $labl$$label;
15119 Assembler::Condition cond =
15120 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15121 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15122 %}
15123 ins_pipe(pipe_cmp_branch);
15124 %}
15125
15126 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15127 match(If cmp (CmpL (AndL op1 op2) op3));
15128 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15129 effect(USE labl);
15130
15131 ins_cost(BRANCH_COST);
15132 format %{ "tb$cmp $op1, $op2, $labl" %}
15133 ins_encode %{
15134 Label* L = $labl$$label;
15135 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15136 int bit = exact_log2_long($op2$$constant);
15137 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15138 %}
15139 ins_pipe(pipe_cmp_branch);
15140 %}
15141
15142 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15143 match(If cmp (CmpI (AndI op1 op2) op3));
15144 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15145 effect(USE labl);
15146
15147 ins_cost(BRANCH_COST);
15148 format %{ "tb$cmp $op1, $op2, $labl" %}
15149 ins_encode %{
15150 Label* L = $labl$$label;
15151 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15152 int bit = exact_log2((juint)$op2$$constant);
15153 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15154 %}
15155 ins_pipe(pipe_cmp_branch);
15156 %}
15157
15158 // Test bits
15159
15160 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15161 match(Set cr (CmpL (AndL op1 op2) op3));
15162 predicate(Assembler::operand_valid_for_logical_immediate
15163 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15164
15165 ins_cost(INSN_COST);
15166 format %{ "tst $op1, $op2 # long" %}
15167 ins_encode %{
15168 __ tst($op1$$Register, $op2$$constant);
15169 %}
15170 ins_pipe(ialu_reg_reg);
15171 %}
15172
15173 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15174 match(Set cr (CmpI (AndI op1 op2) op3));
15175 predicate(Assembler::operand_valid_for_logical_immediate
15176 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15177
15178 ins_cost(INSN_COST);
15179 format %{ "tst $op1, $op2 # int" %}
15180 ins_encode %{
15181 __ tstw($op1$$Register, $op2$$constant);
15182 %}
15183 ins_pipe(ialu_reg_reg);
15184 %}
15185
15186 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15187 match(Set cr (CmpL (AndL op1 op2) op3));
15188
15189 ins_cost(INSN_COST);
15190 format %{ "tst $op1, $op2 # long" %}
15191 ins_encode %{
15192 __ tst($op1$$Register, $op2$$Register);
15193 %}
15194 ins_pipe(ialu_reg_reg);
15195 %}
15196
15197 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15198 match(Set cr (CmpI (AndI op1 op2) op3));
15199
15200 ins_cost(INSN_COST);
15201 format %{ "tstw $op1, $op2 # int" %}
15202 ins_encode %{
15203 __ tstw($op1$$Register, $op2$$Register);
15204 %}
15205 ins_pipe(ialu_reg_reg);
15206 %}
15207
15208
15209 // Conditional Far Branch
15210 // Conditional Far Branch Unsigned
15211 // TODO: fixme
15212
15213 // counted loop end branch near
15214 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15215 %{
15216 match(CountedLoopEnd cmp cr);
15217
15218 effect(USE lbl);
15219
15220 ins_cost(BRANCH_COST);
15221 // short variant.
15222 // ins_short_branch(1);
15223 format %{ "b$cmp $lbl \t// counted loop end" %}
15224
15225 ins_encode(aarch64_enc_br_con(cmp, lbl));
15226
15227 ins_pipe(pipe_branch);
15228 %}
15229
15230 // counted loop end branch far
15231 // TODO: fixme
15232
15233 // ============================================================================
15234 // inlined locking and unlocking
15235
15236 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15237 %{
15238 match(Set cr (FastLock object box));
15239 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15240
15241 ins_cost(5 * INSN_COST);
15242 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15243
15244 ins_encode %{
15245 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15246 %}
15247
15248 ins_pipe(pipe_serial);
15249 %}
15250
15251 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15252 %{
15253 match(Set cr (FastUnlock object box));
15254 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15255
15256 ins_cost(5 * INSN_COST);
15257 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15258
15259 ins_encode %{
15260 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15261 %}
15262
15263 ins_pipe(pipe_serial);
15264 %}
15265
15266 // ============================================================================
15267 // Safepoint Instructions
15268
15269 // TODO
15270 // provide a near and far version of this code
15271
15272 instruct safePoint(rFlagsReg cr, iRegP poll)
15273 %{
15274 match(SafePoint poll);
15275 effect(KILL cr);
15276
15277 format %{
15278 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15279 %}
15280 ins_encode %{
15281 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15282 %}
15283 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15284 %}
15285
15286
15287 // ============================================================================
15288 // Procedure Call/Return Instructions
15289
15290 // Call Java Static Instruction
15291
15292 instruct CallStaticJavaDirect(method meth)
15293 %{
15294 match(CallStaticJava);
15295
15296 effect(USE meth);
15297
15298 ins_cost(CALL_COST);
15299
15300 format %{ "call,static $meth \t// ==> " %}
15301
15302 ins_encode(aarch64_enc_java_static_call(meth),
15303 aarch64_enc_call_epilog);
15304
15305 ins_pipe(pipe_class_call);
15306 %}
15307
15308 // TO HERE
15309
15310 // Call Java Dynamic Instruction
15311 instruct CallDynamicJavaDirect(method meth)
15312 %{
15313 match(CallDynamicJava);
15314
15315 effect(USE meth);
15316
15317 ins_cost(CALL_COST);
15318
15319 format %{ "CALL,dynamic $meth \t// ==> " %}
15320
15321 ins_encode(aarch64_enc_java_dynamic_call(meth),
15322 aarch64_enc_call_epilog);
15323
15324 ins_pipe(pipe_class_call);
15325 %}
15326
15327 // Call Runtime Instruction
15328
15329 instruct CallRuntimeDirect(method meth)
15330 %{
15331 match(CallRuntime);
15332
15333 effect(USE meth);
15334
15335 ins_cost(CALL_COST);
15336
15337 format %{ "CALL, runtime $meth" %}
15338
15339 ins_encode( aarch64_enc_java_to_runtime(meth) );
15340
15341 ins_pipe(pipe_class_call);
15342 %}
15343
15344 // Call Runtime Instruction
15345
15346 instruct CallLeafDirect(method meth)
15347 %{
15348 match(CallLeaf);
15349
15350 effect(USE meth);
15351
15352 ins_cost(CALL_COST);
15353
15354 format %{ "CALL, runtime leaf $meth" %}
15355
15356 ins_encode( aarch64_enc_java_to_runtime(meth) );
15357
15358 ins_pipe(pipe_class_call);
15359 %}
15360
15361 // Call Runtime Instruction without safepoint and with vector arguments
15362 instruct CallLeafDirectVector(method meth)
15363 %{
15364 match(CallLeafVector);
15365
15366 effect(USE meth);
15367
15368 ins_cost(CALL_COST);
15369
15370 format %{ "CALL, runtime leaf vector $meth" %}
15371
15372 ins_encode(aarch64_enc_java_to_runtime(meth));
15373
15374 ins_pipe(pipe_class_call);
15375 %}
15376
15377 // Call Runtime Instruction
15378
15379 instruct CallLeafNoFPDirect(method meth)
15380 %{
15381 match(CallLeafNoFP);
15382
15383 effect(USE meth);
15384
15385 ins_cost(CALL_COST);
15386
15387 format %{ "CALL, runtime leaf nofp $meth" %}
15388
15389 ins_encode( aarch64_enc_java_to_runtime(meth) );
15390
15391 ins_pipe(pipe_class_call);
15392 %}
15393
15394 // Tail Call; Jump from runtime stub to Java code.
15395 // Also known as an 'interprocedural jump'.
15396 // Target of jump will eventually return to caller.
15397 // TailJump below removes the return address.
15398 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15399 // emitted just above the TailCall which has reset rfp to the caller state.
15400 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15401 %{
15402 match(TailCall jump_target method_ptr);
15403
15404 ins_cost(CALL_COST);
15405
15406 format %{ "br $jump_target\t# $method_ptr holds method" %}
15407
15408 ins_encode(aarch64_enc_tail_call(jump_target));
15409
15410 ins_pipe(pipe_class_call);
15411 %}
15412
15413 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15414 %{
15415 match(TailJump jump_target ex_oop);
15416
15417 ins_cost(CALL_COST);
15418
15419 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15420
15421 ins_encode(aarch64_enc_tail_jmp(jump_target));
15422
15423 ins_pipe(pipe_class_call);
15424 %}
15425
15426 // Forward exception.
15427 instruct ForwardExceptionjmp()
15428 %{
15429 match(ForwardException);
15430 ins_cost(CALL_COST);
15431
15432 format %{ "b forward_exception_stub" %}
15433 ins_encode %{
15434 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15435 %}
15436 ins_pipe(pipe_class_call);
15437 %}
15438
15439 // Create exception oop: created by stack-crawling runtime code.
15440 // Created exception is now available to this handler, and is setup
15441 // just prior to jumping to this handler. No code emitted.
15442 // TODO check
15443 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15444 instruct CreateException(iRegP_R0 ex_oop)
15445 %{
15446 match(Set ex_oop (CreateEx));
15447
15448 format %{ " -- \t// exception oop; no code emitted" %}
15449
15450 size(0);
15451
15452 ins_encode( /*empty*/ );
15453
15454 ins_pipe(pipe_class_empty);
15455 %}
15456
15457 // Rethrow exception: The exception oop will come in the first
15458 // argument position. Then JUMP (not call) to the rethrow stub code.
15459 instruct RethrowException() %{
15460 match(Rethrow);
15461 ins_cost(CALL_COST);
15462
15463 format %{ "b rethrow_stub" %}
15464
15465 ins_encode( aarch64_enc_rethrow() );
15466
15467 ins_pipe(pipe_class_call);
15468 %}
15469
15470
15471 // Return Instruction
15472 // epilog node loads ret address into lr as part of frame pop
15473 instruct Ret()
15474 %{
15475 match(Return);
15476
15477 format %{ "ret\t// return register" %}
15478
15479 ins_encode( aarch64_enc_ret() );
15480
15481 ins_pipe(pipe_branch);
15482 %}
15483
15484 // Die now.
15485 instruct ShouldNotReachHere() %{
15486 match(Halt);
15487
15488 ins_cost(CALL_COST);
15489 format %{ "ShouldNotReachHere" %}
15490
15491 ins_encode %{
15492 if (is_reachable()) {
15493 const char* str = __ code_string(_halt_reason);
15494 __ stop(str);
15495 }
15496 %}
15497
15498 ins_pipe(pipe_class_default);
15499 %}
15500
15501 // ============================================================================
15502 // Partial Subtype Check
15503 //
15504 // superklass array for an instance of the superklass. Set a hidden
15505 // internal cache on a hit (cache is checked with exposed code in
15506 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15507 // encoding ALSO sets flags.
15508
15509 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15510 %{
15511 match(Set result (PartialSubtypeCheck sub super));
15512 predicate(!UseSecondarySupersTable);
15513 effect(KILL cr, KILL temp);
15514
15515 ins_cost(20 * INSN_COST); // slightly larger than the next version
15516 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15517
15518 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15519
15520 opcode(0x1); // Force zero of result reg on hit
15521
15522 ins_pipe(pipe_class_memory);
15523 %}
15524
15525 // Two versions of partialSubtypeCheck, both used when we need to
15526 // search for a super class in the secondary supers array. The first
15527 // is used when we don't know _a priori_ the class being searched
15528 // for. The second, far more common, is used when we do know: this is
15529 // used for instanceof, checkcast, and any case where C2 can determine
15530 // it by constant propagation.
15531
15532 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15533 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15534 rFlagsReg cr)
15535 %{
15536 match(Set result (PartialSubtypeCheck sub super));
15537 predicate(UseSecondarySupersTable);
15538 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15539
15540 ins_cost(10 * INSN_COST); // slightly larger than the next version
15541 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15542
15543 ins_encode %{
15544 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15545 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15546 $vtemp$$FloatRegister,
15547 $result$$Register, /*L_success*/nullptr);
15548 %}
15549
15550 ins_pipe(pipe_class_memory);
15551 %}
15552
15553 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15554 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15555 rFlagsReg cr)
15556 %{
15557 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15558 predicate(UseSecondarySupersTable);
15559 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15560
15561 ins_cost(5 * INSN_COST); // smaller than the next version
15562 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15563
15564 ins_encode %{
15565 bool success = false;
15566 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15567 if (InlineSecondarySupersTest) {
15568 success =
15569 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15570 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15571 $vtemp$$FloatRegister,
15572 $result$$Register,
15573 super_klass_slot);
15574 } else {
15575 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15576 success = (call != nullptr);
15577 }
15578 if (!success) {
15579 ciEnv::current()->record_failure("CodeCache is full");
15580 return;
15581 }
15582 %}
15583
15584 ins_pipe(pipe_class_memory);
15585 %}
15586
15587 // Intrisics for String.compareTo()
15588
15589 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15590 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15591 %{
15592 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15593 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15594 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15595
15596 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15597 ins_encode %{
15598 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15599 __ string_compare($str1$$Register, $str2$$Register,
15600 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15601 $tmp1$$Register, $tmp2$$Register,
15602 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15603 %}
15604 ins_pipe(pipe_class_memory);
15605 %}
15606
15607 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15608 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15609 %{
15610 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15611 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15612 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15613
15614 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15615 ins_encode %{
15616 __ string_compare($str1$$Register, $str2$$Register,
15617 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15618 $tmp1$$Register, $tmp2$$Register,
15619 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15620 %}
15621 ins_pipe(pipe_class_memory);
15622 %}
15623
15624 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15625 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15626 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15627 %{
15628 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15629 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15630 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15631 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15632
15633 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15634 ins_encode %{
15635 __ string_compare($str1$$Register, $str2$$Register,
15636 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15637 $tmp1$$Register, $tmp2$$Register,
15638 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15639 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15640 %}
15641 ins_pipe(pipe_class_memory);
15642 %}
15643
15644 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15645 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15646 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15647 %{
15648 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15649 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15650 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15651 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15652
15653 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15654 ins_encode %{
15655 __ string_compare($str1$$Register, $str2$$Register,
15656 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15657 $tmp1$$Register, $tmp2$$Register,
15658 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15659 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15660 %}
15661 ins_pipe(pipe_class_memory);
15662 %}
15663
15664 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15665 // these string_compare variants as NEON register type for convenience so that the prototype of
15666 // string_compare can be shared with all variants.
15667
15668 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15669 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15670 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15671 pRegGov_P1 pgtmp2, rFlagsReg cr)
15672 %{
15673 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15674 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15675 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15676 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15677
15678 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15679 ins_encode %{
15680 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15681 __ string_compare($str1$$Register, $str2$$Register,
15682 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15683 $tmp1$$Register, $tmp2$$Register,
15684 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15685 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15686 StrIntrinsicNode::LL);
15687 %}
15688 ins_pipe(pipe_class_memory);
15689 %}
15690
15691 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15692 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15693 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15694 pRegGov_P1 pgtmp2, rFlagsReg cr)
15695 %{
15696 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15697 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15698 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15699 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15700
15701 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15702 ins_encode %{
15703 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15704 __ string_compare($str1$$Register, $str2$$Register,
15705 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15706 $tmp1$$Register, $tmp2$$Register,
15707 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15708 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15709 StrIntrinsicNode::LU);
15710 %}
15711 ins_pipe(pipe_class_memory);
15712 %}
15713
15714 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15715 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15716 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15717 pRegGov_P1 pgtmp2, rFlagsReg cr)
15718 %{
15719 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15720 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15721 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15722 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15723
15724 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15725 ins_encode %{
15726 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15727 __ string_compare($str1$$Register, $str2$$Register,
15728 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15729 $tmp1$$Register, $tmp2$$Register,
15730 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15731 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15732 StrIntrinsicNode::UL);
15733 %}
15734 ins_pipe(pipe_class_memory);
15735 %}
15736
15737 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15738 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15739 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15740 pRegGov_P1 pgtmp2, rFlagsReg cr)
15741 %{
15742 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15743 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15744 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15745 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15746
15747 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15748 ins_encode %{
15749 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15750 __ string_compare($str1$$Register, $str2$$Register,
15751 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15752 $tmp1$$Register, $tmp2$$Register,
15753 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15754 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15755 StrIntrinsicNode::UU);
15756 %}
15757 ins_pipe(pipe_class_memory);
15758 %}
15759
15760 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15761 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15762 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15763 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15764 %{
15765 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15766 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15767 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15768 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15769 TEMP vtmp0, TEMP vtmp1, KILL cr);
15770 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15771 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15772
15773 ins_encode %{
15774 __ string_indexof($str1$$Register, $str2$$Register,
15775 $cnt1$$Register, $cnt2$$Register,
15776 $tmp1$$Register, $tmp2$$Register,
15777 $tmp3$$Register, $tmp4$$Register,
15778 $tmp5$$Register, $tmp6$$Register,
15779 -1, $result$$Register, StrIntrinsicNode::UU);
15780 %}
15781 ins_pipe(pipe_class_memory);
15782 %}
15783
15784 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15785 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15786 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15787 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15788 %{
15789 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15790 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15791 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15792 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15793 TEMP vtmp0, TEMP vtmp1, KILL cr);
15794 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15795 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15796
15797 ins_encode %{
15798 __ string_indexof($str1$$Register, $str2$$Register,
15799 $cnt1$$Register, $cnt2$$Register,
15800 $tmp1$$Register, $tmp2$$Register,
15801 $tmp3$$Register, $tmp4$$Register,
15802 $tmp5$$Register, $tmp6$$Register,
15803 -1, $result$$Register, StrIntrinsicNode::LL);
15804 %}
15805 ins_pipe(pipe_class_memory);
15806 %}
15807
15808 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15809 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15810 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15811 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15812 %{
15813 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15814 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15815 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15816 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15817 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15818 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15819 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15820
15821 ins_encode %{
15822 __ string_indexof($str1$$Register, $str2$$Register,
15823 $cnt1$$Register, $cnt2$$Register,
15824 $tmp1$$Register, $tmp2$$Register,
15825 $tmp3$$Register, $tmp4$$Register,
15826 $tmp5$$Register, $tmp6$$Register,
15827 -1, $result$$Register, StrIntrinsicNode::UL);
15828 %}
15829 ins_pipe(pipe_class_memory);
15830 %}
15831
15832 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15833 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15834 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15835 %{
15836 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15837 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15838 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15839 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15840 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15841 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15842
15843 ins_encode %{
15844 int icnt2 = (int)$int_cnt2$$constant;
15845 __ string_indexof($str1$$Register, $str2$$Register,
15846 $cnt1$$Register, zr,
15847 $tmp1$$Register, $tmp2$$Register,
15848 $tmp3$$Register, $tmp4$$Register, zr, zr,
15849 icnt2, $result$$Register, StrIntrinsicNode::UU);
15850 %}
15851 ins_pipe(pipe_class_memory);
15852 %}
15853
15854 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15855 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15856 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15857 %{
15858 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15859 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15860 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15861 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15862 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15863 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15864
15865 ins_encode %{
15866 int icnt2 = (int)$int_cnt2$$constant;
15867 __ string_indexof($str1$$Register, $str2$$Register,
15868 $cnt1$$Register, zr,
15869 $tmp1$$Register, $tmp2$$Register,
15870 $tmp3$$Register, $tmp4$$Register, zr, zr,
15871 icnt2, $result$$Register, StrIntrinsicNode::LL);
15872 %}
15873 ins_pipe(pipe_class_memory);
15874 %}
15875
15876 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15877 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15878 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15879 %{
15880 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15881 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15882 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15883 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15884 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15885 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15886
15887 ins_encode %{
15888 int icnt2 = (int)$int_cnt2$$constant;
15889 __ string_indexof($str1$$Register, $str2$$Register,
15890 $cnt1$$Register, zr,
15891 $tmp1$$Register, $tmp2$$Register,
15892 $tmp3$$Register, $tmp4$$Register, zr, zr,
15893 icnt2, $result$$Register, StrIntrinsicNode::UL);
15894 %}
15895 ins_pipe(pipe_class_memory);
15896 %}
15897
15898 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15899 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15900 iRegINoSp tmp3, rFlagsReg cr)
15901 %{
15902 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15903 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15904 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15905 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15906
15907 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15908
15909 ins_encode %{
15910 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15911 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15912 $tmp3$$Register);
15913 %}
15914 ins_pipe(pipe_class_memory);
15915 %}
15916
15917 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15918 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15919 iRegINoSp tmp3, rFlagsReg cr)
15920 %{
15921 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15922 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15923 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15924 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15925
15926 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15927
15928 ins_encode %{
15929 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15930 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15931 $tmp3$$Register);
15932 %}
15933 ins_pipe(pipe_class_memory);
15934 %}
15935
15936 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15937 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15938 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15939 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15940 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15941 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15942 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15943 ins_encode %{
15944 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15945 $result$$Register, $ztmp1$$FloatRegister,
15946 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15947 $ptmp$$PRegister, true /* isL */);
15948 %}
15949 ins_pipe(pipe_class_memory);
15950 %}
15951
15952 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15953 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15954 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15955 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
15956 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15957 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15958 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15959 ins_encode %{
15960 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15961 $result$$Register, $ztmp1$$FloatRegister,
15962 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15963 $ptmp$$PRegister, false /* isL */);
15964 %}
15965 ins_pipe(pipe_class_memory);
15966 %}
15967
15968 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15969 iRegI_R0 result, rFlagsReg cr)
15970 %{
15971 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
15972 match(Set result (StrEquals (Binary str1 str2) cnt));
15973 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15974
15975 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15976 ins_encode %{
15977 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15978 __ string_equals($str1$$Register, $str2$$Register,
15979 $result$$Register, $cnt$$Register);
15980 %}
15981 ins_pipe(pipe_class_memory);
15982 %}
15983
15984 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
15985 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
15986 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
15987 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
15988 iRegP_R10 tmp, rFlagsReg cr)
15989 %{
15990 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15991 match(Set result (AryEq ary1 ary2));
15992 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
15993 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
15994 TEMP vtmp6, TEMP vtmp7, KILL cr);
15995
15996 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
15997 ins_encode %{
15998 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
15999 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16000 $result$$Register, $tmp$$Register, 1);
16001 if (tpc == nullptr) {
16002 ciEnv::current()->record_failure("CodeCache is full");
16003 return;
16004 }
16005 %}
16006 ins_pipe(pipe_class_memory);
16007 %}
16008
16009 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16010 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16011 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16012 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16013 iRegP_R10 tmp, rFlagsReg cr)
16014 %{
16015 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16016 match(Set result (AryEq ary1 ary2));
16017 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16018 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16019 TEMP vtmp6, TEMP vtmp7, KILL cr);
16020
16021 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16022 ins_encode %{
16023 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16024 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16025 $result$$Register, $tmp$$Register, 2);
16026 if (tpc == nullptr) {
16027 ciEnv::current()->record_failure("CodeCache is full");
16028 return;
16029 }
16030 %}
16031 ins_pipe(pipe_class_memory);
16032 %}
16033
16034 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16035 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16036 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16037 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16038 %{
16039 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16040 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16041 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16042
16043 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16044 ins_encode %{
16045 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16046 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16047 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16048 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16049 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16050 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16051 (BasicType)$basic_type$$constant);
16052 if (tpc == nullptr) {
16053 ciEnv::current()->record_failure("CodeCache is full");
16054 return;
16055 }
16056 %}
16057 ins_pipe(pipe_class_memory);
16058 %}
16059
16060 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16061 %{
16062 match(Set result (CountPositives ary1 len));
16063 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16064 format %{ "count positives byte[] $ary1,$len -> $result" %}
16065 ins_encode %{
16066 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16067 if (tpc == nullptr) {
16068 ciEnv::current()->record_failure("CodeCache is full");
16069 return;
16070 }
16071 %}
16072 ins_pipe( pipe_slow );
16073 %}
16074
16075 // fast char[] to byte[] compression
16076 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16077 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16078 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16079 iRegI_R0 result, rFlagsReg cr)
16080 %{
16081 match(Set result (StrCompressedCopy src (Binary dst len)));
16082 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16083 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16084
16085 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16086 ins_encode %{
16087 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16088 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16089 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16090 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16091 %}
16092 ins_pipe(pipe_slow);
16093 %}
16094
16095 // fast byte[] to char[] inflation
16096 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16097 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16098 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16099 %{
16100 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16101 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16102 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16103 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16104
16105 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16106 ins_encode %{
16107 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16108 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16109 $vtmp2$$FloatRegister, $tmp$$Register);
16110 if (tpc == nullptr) {
16111 ciEnv::current()->record_failure("CodeCache is full");
16112 return;
16113 }
16114 %}
16115 ins_pipe(pipe_class_memory);
16116 %}
16117
16118 // encode char[] to byte[] in ISO_8859_1
16119 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16120 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16121 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16122 iRegI_R0 result, rFlagsReg cr)
16123 %{
16124 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16125 match(Set result (EncodeISOArray src (Binary dst len)));
16126 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16127 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16128
16129 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16130 ins_encode %{
16131 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16132 $result$$Register, false,
16133 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16134 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16135 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16136 %}
16137 ins_pipe(pipe_class_memory);
16138 %}
16139
16140 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16141 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16142 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16143 iRegI_R0 result, rFlagsReg cr)
16144 %{
16145 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16146 match(Set result (EncodeISOArray src (Binary dst len)));
16147 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16148 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16149
16150 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16151 ins_encode %{
16152 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16153 $result$$Register, true,
16154 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16155 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16156 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16157 %}
16158 ins_pipe(pipe_class_memory);
16159 %}
16160
16161 //----------------------------- CompressBits/ExpandBits ------------------------
16162
16163 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16164 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16165 match(Set dst (CompressBits src mask));
16166 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16167 format %{ "mov $tsrc, $src\n\t"
16168 "mov $tmask, $mask\n\t"
16169 "bext $tdst, $tsrc, $tmask\n\t"
16170 "mov $dst, $tdst"
16171 %}
16172 ins_encode %{
16173 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16174 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16175 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16176 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16177 %}
16178 ins_pipe(pipe_slow);
16179 %}
16180
16181 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16182 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16183 match(Set dst (CompressBits (LoadI mem) mask));
16184 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16185 format %{ "ldrs $tsrc, $mem\n\t"
16186 "ldrs $tmask, $mask\n\t"
16187 "bext $tdst, $tsrc, $tmask\n\t"
16188 "mov $dst, $tdst"
16189 %}
16190 ins_encode %{
16191 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16192 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16193 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16194 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16195 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16196 %}
16197 ins_pipe(pipe_slow);
16198 %}
16199
16200 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16201 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16202 match(Set dst (CompressBits src mask));
16203 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16204 format %{ "mov $tsrc, $src\n\t"
16205 "mov $tmask, $mask\n\t"
16206 "bext $tdst, $tsrc, $tmask\n\t"
16207 "mov $dst, $tdst"
16208 %}
16209 ins_encode %{
16210 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16211 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16212 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16213 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16214 %}
16215 ins_pipe(pipe_slow);
16216 %}
16217
16218 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16219 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16220 match(Set dst (CompressBits (LoadL mem) mask));
16221 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16222 format %{ "ldrd $tsrc, $mem\n\t"
16223 "ldrd $tmask, $mask\n\t"
16224 "bext $tdst, $tsrc, $tmask\n\t"
16225 "mov $dst, $tdst"
16226 %}
16227 ins_encode %{
16228 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16229 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16230 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16231 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16232 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16233 %}
16234 ins_pipe(pipe_slow);
16235 %}
16236
16237 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16238 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16239 match(Set dst (ExpandBits src mask));
16240 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16241 format %{ "mov $tsrc, $src\n\t"
16242 "mov $tmask, $mask\n\t"
16243 "bdep $tdst, $tsrc, $tmask\n\t"
16244 "mov $dst, $tdst"
16245 %}
16246 ins_encode %{
16247 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16248 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16249 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16250 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16251 %}
16252 ins_pipe(pipe_slow);
16253 %}
16254
16255 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16256 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16257 match(Set dst (ExpandBits (LoadI mem) mask));
16258 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16259 format %{ "ldrs $tsrc, $mem\n\t"
16260 "ldrs $tmask, $mask\n\t"
16261 "bdep $tdst, $tsrc, $tmask\n\t"
16262 "mov $dst, $tdst"
16263 %}
16264 ins_encode %{
16265 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16266 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16267 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16268 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16269 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16270 %}
16271 ins_pipe(pipe_slow);
16272 %}
16273
16274 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16275 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16276 match(Set dst (ExpandBits src mask));
16277 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16278 format %{ "mov $tsrc, $src\n\t"
16279 "mov $tmask, $mask\n\t"
16280 "bdep $tdst, $tsrc, $tmask\n\t"
16281 "mov $dst, $tdst"
16282 %}
16283 ins_encode %{
16284 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16285 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16286 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16287 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16288 %}
16289 ins_pipe(pipe_slow);
16290 %}
16291
16292
16293 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16294 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16295 match(Set dst (ExpandBits (LoadL mem) mask));
16296 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16297 format %{ "ldrd $tsrc, $mem\n\t"
16298 "ldrd $tmask, $mask\n\t"
16299 "bdep $tdst, $tsrc, $tmask\n\t"
16300 "mov $dst, $tdst"
16301 %}
16302 ins_encode %{
16303 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16304 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16305 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16306 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16307 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16308 %}
16309 ins_pipe(pipe_slow);
16310 %}
16311
16312 //----------------------------- Reinterpret ----------------------------------
16313 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16314 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16315 match(Set dst (ReinterpretHF2S src));
16316 format %{ "reinterpretHF2S $dst, $src" %}
16317 ins_encode %{
16318 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16319 %}
16320 ins_pipe(pipe_slow);
16321 %}
16322
16323 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16324 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16325 match(Set dst (ReinterpretS2HF src));
16326 format %{ "reinterpretS2HF $dst, $src" %}
16327 ins_encode %{
16328 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16329 %}
16330 ins_pipe(pipe_slow);
16331 %}
16332
16333 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16334 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16335 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16336 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16337 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16338 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16339 // can be omitted in this pattern, resulting in -
16340 // fcvt $dst, $src // Convert float to half-precision float
16341 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16342 %{
16343 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16344 format %{ "convF2HFAndS2HF $dst, $src" %}
16345 ins_encode %{
16346 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16347 %}
16348 ins_pipe(pipe_slow);
16349 %}
16350
16351 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16352 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16353 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16354 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16355 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16356 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16357 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16358 // resulting in -
16359 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16360 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16361 %{
16362 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16363 format %{ "convHF2SAndHF2F $dst, $src" %}
16364 ins_encode %{
16365 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16366 %}
16367 ins_pipe(pipe_slow);
16368 %}
16369
16370 // ============================================================================
16371 // This name is KNOWN by the ADLC and cannot be changed.
16372 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16373 // for this guy.
16374 instruct tlsLoadP(thread_RegP dst)
16375 %{
16376 match(Set dst (ThreadLocal));
16377
16378 ins_cost(0);
16379
16380 format %{ " -- \t// $dst=Thread::current(), empty" %}
16381
16382 size(0);
16383
16384 ins_encode( /*empty*/ );
16385
16386 ins_pipe(pipe_class_empty);
16387 %}
16388
16389 //----------PEEPHOLE RULES-----------------------------------------------------
16390 // These must follow all instruction definitions as they use the names
16391 // defined in the instructions definitions.
16392 //
16393 // peepmatch ( root_instr_name [preceding_instruction]* );
16394 //
16395 // peepconstraint %{
16396 // (instruction_number.operand_name relational_op instruction_number.operand_name
16397 // [, ...] );
16398 // // instruction numbers are zero-based using left to right order in peepmatch
16399 //
16400 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16401 // // provide an instruction_number.operand_name for each operand that appears
16402 // // in the replacement instruction's match rule
16403 //
16404 // ---------VM FLAGS---------------------------------------------------------
16405 //
16406 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16407 //
16408 // Each peephole rule is given an identifying number starting with zero and
16409 // increasing by one in the order seen by the parser. An individual peephole
16410 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16411 // on the command-line.
16412 //
16413 // ---------CURRENT LIMITATIONS----------------------------------------------
16414 //
16415 // Only match adjacent instructions in same basic block
16416 // Only equality constraints
16417 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16418 // Only one replacement instruction
16419 //
16420 // ---------EXAMPLE----------------------------------------------------------
16421 //
16422 // // pertinent parts of existing instructions in architecture description
16423 // instruct movI(iRegINoSp dst, iRegI src)
16424 // %{
16425 // match(Set dst (CopyI src));
16426 // %}
16427 //
16428 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16429 // %{
16430 // match(Set dst (AddI dst src));
16431 // effect(KILL cr);
16432 // %}
16433 //
16434 // // Change (inc mov) to lea
16435 // peephole %{
16436 // // increment preceded by register-register move
16437 // peepmatch ( incI_iReg movI );
16438 // // require that the destination register of the increment
16439 // // match the destination register of the move
16440 // peepconstraint ( 0.dst == 1.dst );
16441 // // construct a replacement instruction that sets
16442 // // the destination to ( move's source register + one )
16443 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16444 // %}
16445 //
16446
16447 // Implementation no longer uses movX instructions since
16448 // machine-independent system no longer uses CopyX nodes.
16449 //
16450 // peephole
16451 // %{
16452 // peepmatch (incI_iReg movI);
16453 // peepconstraint (0.dst == 1.dst);
16454 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16455 // %}
16456
16457 // peephole
16458 // %{
16459 // peepmatch (decI_iReg movI);
16460 // peepconstraint (0.dst == 1.dst);
16461 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16462 // %}
16463
16464 // peephole
16465 // %{
16466 // peepmatch (addI_iReg_imm movI);
16467 // peepconstraint (0.dst == 1.dst);
16468 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16469 // %}
16470
16471 // peephole
16472 // %{
16473 // peepmatch (incL_iReg movL);
16474 // peepconstraint (0.dst == 1.dst);
16475 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16476 // %}
16477
16478 // peephole
16479 // %{
16480 // peepmatch (decL_iReg movL);
16481 // peepconstraint (0.dst == 1.dst);
16482 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16483 // %}
16484
16485 // peephole
16486 // %{
16487 // peepmatch (addL_iReg_imm movL);
16488 // peepconstraint (0.dst == 1.dst);
16489 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16490 // %}
16491
16492 // peephole
16493 // %{
16494 // peepmatch (addP_iReg_imm movP);
16495 // peepconstraint (0.dst == 1.dst);
16496 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16497 // %}
16498
16499 // // Change load of spilled value to only a spill
16500 // instruct storeI(memory mem, iRegI src)
16501 // %{
16502 // match(Set mem (StoreI mem src));
16503 // %}
16504 //
16505 // instruct loadI(iRegINoSp dst, memory mem)
16506 // %{
16507 // match(Set dst (LoadI mem));
16508 // %}
16509 //
16510
16511 //----------SMARTSPILL RULES---------------------------------------------------
16512 // These must follow all instruction definitions as they use the names
16513 // defined in the instructions definitions.
16514
16515 // Local Variables:
16516 // mode: c++
16517 // End: