1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_CompareAndSwapB:
1512 case Op_CompareAndSwapS:
1513 case Op_GetAndSetI:
1514 case Op_GetAndSetL:
1515 case Op_GetAndSetP:
1516 case Op_GetAndSetN:
1517 case Op_GetAndAddI:
1518 case Op_GetAndAddL:
1519 return true;
1520 case Op_CompareAndExchangeI:
1521 case Op_CompareAndExchangeN:
1522 case Op_CompareAndExchangeB:
1523 case Op_CompareAndExchangeS:
1524 case Op_CompareAndExchangeL:
1525 case Op_CompareAndExchangeP:
1526 case Op_WeakCompareAndSwapB:
1527 case Op_WeakCompareAndSwapS:
1528 case Op_WeakCompareAndSwapI:
1529 case Op_WeakCompareAndSwapL:
1530 case Op_WeakCompareAndSwapP:
1531 case Op_WeakCompareAndSwapN:
1532 return maybe_volatile;
1533 default:
1534 return false;
1535 }
1536 }
1537
1538 // helper to determine the maximum number of Phi nodes we may need to
1539 // traverse when searching from a card mark membar for the merge mem
1540 // feeding a trailing membar or vice versa
1541
1542 // predicates controlling emit of ldr<x>/ldar<x>
1543
1544 bool unnecessary_acquire(const Node *barrier)
1545 {
1546 assert(barrier->is_MemBar(), "expecting a membar");
1547
1548 MemBarNode* mb = barrier->as_MemBar();
1549
1550 if (mb->trailing_load()) {
1551 return true;
1552 }
1553
1554 if (mb->trailing_load_store()) {
1555 Node* load_store = mb->in(MemBarNode::Precedent);
1556 assert(load_store->is_LoadStore(), "unexpected graph shape");
1557 return is_CAS(load_store->Opcode(), true);
1558 }
1559
1560 return false;
1561 }
1562
1563 bool needs_acquiring_load(const Node *n)
1564 {
1565 assert(n->is_Load(), "expecting a load");
1566 LoadNode *ld = n->as_Load();
1567 return ld->is_acquire();
1568 }
1569
1570 bool unnecessary_release(const Node *n)
1571 {
1572 assert((n->is_MemBar() &&
1573 n->Opcode() == Op_MemBarRelease),
1574 "expecting a release membar");
1575
1576 MemBarNode *barrier = n->as_MemBar();
1577 if (!barrier->leading()) {
1578 return false;
1579 } else {
1580 Node* trailing = barrier->trailing_membar();
1581 MemBarNode* trailing_mb = trailing->as_MemBar();
1582 assert(trailing_mb->trailing(), "Not a trailing membar?");
1583 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1584
1585 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1586 if (mem->is_Store()) {
1587 assert(mem->as_Store()->is_release(), "");
1588 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1589 return true;
1590 } else {
1591 assert(mem->is_LoadStore(), "");
1592 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1593 return is_CAS(mem->Opcode(), true);
1594 }
1595 }
1596 return false;
1597 }
1598
1599 bool unnecessary_volatile(const Node *n)
1600 {
1601 // assert n->is_MemBar();
1602 MemBarNode *mbvol = n->as_MemBar();
1603
1604 bool release = mbvol->trailing_store();
1605 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1606 #ifdef ASSERT
1607 if (release) {
1608 Node* leading = mbvol->leading_membar();
1609 assert(leading->Opcode() == Op_MemBarRelease, "");
1610 assert(leading->as_MemBar()->leading_store(), "");
1611 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1612 }
1613 #endif
1614
1615 return release;
1616 }
1617
1618 // predicates controlling emit of str<x>/stlr<x>
1619
1620 bool needs_releasing_store(const Node *n)
1621 {
1622 // assert n->is_Store();
1623 StoreNode *st = n->as_Store();
1624 return st->trailing_membar() != nullptr;
1625 }
1626
1627 // predicate controlling translation of CAS
1628 //
1629 // returns true if CAS needs to use an acquiring load otherwise false
1630
1631 bool needs_acquiring_load_exclusive(const Node *n)
1632 {
1633 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1634 LoadStoreNode* ldst = n->as_LoadStore();
1635 if (is_CAS(n->Opcode(), false)) {
1636 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1637 } else {
1638 return ldst->trailing_membar() != nullptr;
1639 }
1640
1641 // so we can just return true here
1642 return true;
1643 }
1644
1645 #define __ masm->
1646
1647 // advance declarations for helper functions to convert register
1648 // indices to register objects
1649
1650 // the ad file has to provide implementations of certain methods
1651 // expected by the generic code
1652 //
1653 // REQUIRED FUNCTIONALITY
1654
1655 //=============================================================================
1656
1657 // !!!!! Special hack to get all types of calls to specify the byte offset
1658 // from the start of the call to the point where the return address
1659 // will point.
1660
1661 int MachCallStaticJavaNode::ret_addr_offset()
1662 {
1663 // call should be a simple bl
1664 int off = 4;
1665 return off;
1666 }
1667
1668 int MachCallDynamicJavaNode::ret_addr_offset()
1669 {
1670 return 16; // movz, movk, movk, bl
1671 }
1672
1673 int MachCallRuntimeNode::ret_addr_offset() {
1674 // for generated stubs the call will be
1675 // bl(addr)
1676 // or with far branches
1677 // bl(trampoline_stub)
1678 // for real runtime callouts it will be six instructions
1679 // see aarch64_enc_java_to_runtime
1680 // adr(rscratch2, retaddr)
1681 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1682 // lea(rscratch1, RuntimeAddress(addr)
1683 // blr(rscratch1)
1684 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1685 if (cb) {
1686 return 1 * NativeInstruction::instruction_size;
1687 } else {
1688 return 6 * NativeInstruction::instruction_size;
1689 }
1690 }
1691
1692 //=============================================================================
1693
1694 #ifndef PRODUCT
1695 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1696 st->print("BREAKPOINT");
1697 }
1698 #endif
1699
1700 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1701 __ brk(0);
1702 }
1703
1704 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1705 return MachNode::size(ra_);
1706 }
1707
1708 //=============================================================================
1709
1710 #ifndef PRODUCT
1711 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1712 st->print("nop \t# %d bytes pad for loops and calls", _count);
1713 }
1714 #endif
1715
1716 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1717 for (int i = 0; i < _count; i++) {
1718 __ nop();
1719 }
1720 }
1721
1722 uint MachNopNode::size(PhaseRegAlloc*) const {
1723 return _count * NativeInstruction::instruction_size;
1724 }
1725
1726 //=============================================================================
1727 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1728
1729 int ConstantTable::calculate_table_base_offset() const {
1730 return 0; // absolute addressing, no offset
1731 }
1732
1733 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1734 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1735 ShouldNotReachHere();
1736 }
1737
1738 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1739 // Empty encoding
1740 }
1741
1742 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1743 return 0;
1744 }
1745
1746 #ifndef PRODUCT
1747 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1748 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1749 }
1750 #endif
1751
1752 #ifndef PRODUCT
1753 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1754 Compile* C = ra_->C;
1755
1756 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1757
1758 if (C->output()->need_stack_bang(framesize))
1759 st->print("# stack bang size=%d\n\t", framesize);
1760
1761 if (VM_Version::use_rop_protection()) {
1762 st->print("ldr zr, [lr]\n\t");
1763 st->print("paciaz\n\t");
1764 }
1765 if (framesize < ((1 << 9) + 2 * wordSize)) {
1766 st->print("sub sp, sp, #%d\n\t", framesize);
1767 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1768 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1769 } else {
1770 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1771 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1772 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1773 st->print("sub sp, sp, rscratch1");
1774 }
1775 if (C->stub_function() == nullptr) {
1776 st->print("\n\t");
1777 st->print("ldr rscratch1, [guard]\n\t");
1778 st->print("dmb ishld\n\t");
1779 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1780 st->print("cmp rscratch1, rscratch2\n\t");
1781 st->print("b.eq skip");
1782 st->print("\n\t");
1783 st->print("blr #nmethod_entry_barrier_stub\n\t");
1784 st->print("b skip\n\t");
1785 st->print("guard: int\n\t");
1786 st->print("\n\t");
1787 st->print("skip:\n\t");
1788 }
1789 }
1790 #endif
1791
1792 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1793 Compile* C = ra_->C;
1794
1795 // n.b. frame size includes space for return pc and rfp
1796 const int framesize = C->output()->frame_size_in_bytes();
1797
1798 if (C->clinit_barrier_on_entry()) {
1799 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1800
1801 Label L_skip_barrier;
1802
1803 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1804 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1805 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1806 __ bind(L_skip_barrier);
1807 }
1808
1809 if (C->max_vector_size() > 0) {
1810 __ reinitialize_ptrue();
1811 }
1812
1813 int bangsize = C->output()->bang_size_in_bytes();
1814 if (C->output()->need_stack_bang(bangsize))
1815 __ generate_stack_overflow_check(bangsize);
1816
1817 __ build_frame(framesize);
1818
1819 if (C->stub_function() == nullptr) {
1820 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1821 // Dummy labels for just measuring the code size
1822 Label dummy_slow_path;
1823 Label dummy_continuation;
1824 Label dummy_guard;
1825 Label* slow_path = &dummy_slow_path;
1826 Label* continuation = &dummy_continuation;
1827 Label* guard = &dummy_guard;
1828 if (!Compile::current()->output()->in_scratch_emit_size()) {
1829 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1830 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1831 Compile::current()->output()->add_stub(stub);
1832 slow_path = &stub->entry();
1833 continuation = &stub->continuation();
1834 guard = &stub->guard();
1835 }
1836 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1837 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1838 }
1839
1840 if (VerifyStackAtCalls) {
1841 Unimplemented();
1842 }
1843
1844 C->output()->set_frame_complete(__ offset());
1845
1846 if (C->has_mach_constant_base_node()) {
1847 // NOTE: We set the table base offset here because users might be
1848 // emitted before MachConstantBaseNode.
1849 ConstantTable& constant_table = C->output()->constant_table();
1850 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1851 }
1852 }
1853
1854 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1855 {
1856 return MachNode::size(ra_); // too many variables; just compute it
1857 // the hard way
1858 }
1859
1860 int MachPrologNode::reloc() const
1861 {
1862 return 0;
1863 }
1864
1865 //=============================================================================
1866
1867 #ifndef PRODUCT
1868 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1869 Compile* C = ra_->C;
1870 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1871
1872 st->print("# pop frame %d\n\t",framesize);
1873
1874 if (framesize == 0) {
1875 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1876 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1877 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1878 st->print("add sp, sp, #%d\n\t", framesize);
1879 } else {
1880 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1881 st->print("add sp, sp, rscratch1\n\t");
1882 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1883 }
1884 if (VM_Version::use_rop_protection()) {
1885 st->print("autiaz\n\t");
1886 st->print("ldr zr, [lr]\n\t");
1887 }
1888
1889 if (do_polling() && C->is_method_compilation()) {
1890 st->print("# test polling word\n\t");
1891 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1892 st->print("cmp sp, rscratch1\n\t");
1893 st->print("bhi #slow_path");
1894 }
1895 }
1896 #endif
1897
1898 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1899 Compile* C = ra_->C;
1900 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1901
1902 __ remove_frame(framesize);
1903
1904 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1905 __ reserved_stack_check();
1906 }
1907
1908 if (do_polling() && C->is_method_compilation()) {
1909 Label dummy_label;
1910 Label* code_stub = &dummy_label;
1911 if (!C->output()->in_scratch_emit_size()) {
1912 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1913 C->output()->add_stub(stub);
1914 code_stub = &stub->entry();
1915 }
1916 __ relocate(relocInfo::poll_return_type);
1917 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1918 }
1919 }
1920
1921 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1922 // Variable size. Determine dynamically.
1923 return MachNode::size(ra_);
1924 }
1925
1926 int MachEpilogNode::reloc() const {
1927 // Return number of relocatable values contained in this instruction.
1928 return 1; // 1 for polling page.
1929 }
1930
1931 const Pipeline * MachEpilogNode::pipeline() const {
1932 return MachNode::pipeline_class();
1933 }
1934
1935 //=============================================================================
1936
1937 static enum RC rc_class(OptoReg::Name reg) {
1938
1939 if (reg == OptoReg::Bad) {
1940 return rc_bad;
1941 }
1942
1943 // we have 32 int registers * 2 halves
1944 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1945
1946 if (reg < slots_of_int_registers) {
1947 return rc_int;
1948 }
1949
1950 // we have 32 float register * 8 halves
1951 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1952 if (reg < slots_of_int_registers + slots_of_float_registers) {
1953 return rc_float;
1954 }
1955
1956 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1957 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1958 return rc_predicate;
1959 }
1960
1961 // Between predicate regs & stack is the flags.
1962 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1963
1964 return rc_stack;
1965 }
1966
1967 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1968 Compile* C = ra_->C;
1969
1970 // Get registers to move.
1971 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1972 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1973 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1974 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1975
1976 enum RC src_hi_rc = rc_class(src_hi);
1977 enum RC src_lo_rc = rc_class(src_lo);
1978 enum RC dst_hi_rc = rc_class(dst_hi);
1979 enum RC dst_lo_rc = rc_class(dst_lo);
1980
1981 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1982
1983 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1984 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1985 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1986 "expected aligned-adjacent pairs");
1987 }
1988
1989 if (src_lo == dst_lo && src_hi == dst_hi) {
1990 return 0; // Self copy, no move.
1991 }
1992
1993 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1994 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1995 int src_offset = ra_->reg2offset(src_lo);
1996 int dst_offset = ra_->reg2offset(dst_lo);
1997
1998 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1999 uint ireg = ideal_reg();
2000 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2001 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2002 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2003 if (ireg == Op_VecA && masm) {
2004 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2005 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2006 // stack->stack
2007 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2008 sve_vector_reg_size_in_bytes);
2009 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2010 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2011 sve_vector_reg_size_in_bytes);
2012 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2013 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2016 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]),
2018 as_FloatRegister(Matcher::_regEncode[src_lo]));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 } else if (masm) {
2023 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2024 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2025 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2026 // stack->stack
2027 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2028 if (ireg == Op_VecD) {
2029 __ unspill(rscratch1, true, src_offset);
2030 __ spill(rscratch1, true, dst_offset);
2031 } else {
2032 __ spill_copy128(src_offset, dst_offset);
2033 }
2034 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2035 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2036 ireg == Op_VecD ? __ T8B : __ T16B,
2037 as_FloatRegister(Matcher::_regEncode[src_lo]));
2038 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2039 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2040 ireg == Op_VecD ? __ D : __ Q,
2041 ra_->reg2offset(dst_lo));
2042 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2043 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2044 ireg == Op_VecD ? __ D : __ Q,
2045 ra_->reg2offset(src_lo));
2046 } else {
2047 ShouldNotReachHere();
2048 }
2049 }
2050 } else if (masm) {
2051 switch (src_lo_rc) {
2052 case rc_int:
2053 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2054 if (is64) {
2055 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2056 as_Register(Matcher::_regEncode[src_lo]));
2057 } else {
2058 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2059 as_Register(Matcher::_regEncode[src_lo]));
2060 }
2061 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2062 if (is64) {
2063 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2064 as_Register(Matcher::_regEncode[src_lo]));
2065 } else {
2066 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2067 as_Register(Matcher::_regEncode[src_lo]));
2068 }
2069 } else { // gpr --> stack spill
2070 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2071 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2072 }
2073 break;
2074 case rc_float:
2075 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2076 if (is64) {
2077 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2078 as_FloatRegister(Matcher::_regEncode[src_lo]));
2079 } else {
2080 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2081 as_FloatRegister(Matcher::_regEncode[src_lo]));
2082 }
2083 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2084 if (is64) {
2085 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2086 as_FloatRegister(Matcher::_regEncode[src_lo]));
2087 } else {
2088 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2089 as_FloatRegister(Matcher::_regEncode[src_lo]));
2090 }
2091 } else { // fpr --> stack spill
2092 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2093 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2094 is64 ? __ D : __ S, dst_offset);
2095 }
2096 break;
2097 case rc_stack:
2098 if (dst_lo_rc == rc_int) { // stack --> gpr load
2099 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2100 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2101 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2102 is64 ? __ D : __ S, src_offset);
2103 } else if (dst_lo_rc == rc_predicate) {
2104 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2105 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2106 } else { // stack --> stack copy
2107 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2108 if (ideal_reg() == Op_RegVectMask) {
2109 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2110 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2111 } else {
2112 __ unspill(rscratch1, is64, src_offset);
2113 __ spill(rscratch1, is64, dst_offset);
2114 }
2115 }
2116 break;
2117 case rc_predicate:
2118 if (dst_lo_rc == rc_predicate) {
2119 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2120 } else if (dst_lo_rc == rc_stack) {
2121 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2122 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2123 } else {
2124 assert(false, "bad src and dst rc_class combination.");
2125 ShouldNotReachHere();
2126 }
2127 break;
2128 default:
2129 assert(false, "bad rc_class for spill");
2130 ShouldNotReachHere();
2131 }
2132 }
2133
2134 if (st) {
2135 st->print("spill ");
2136 if (src_lo_rc == rc_stack) {
2137 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2138 } else {
2139 st->print("%s -> ", Matcher::regName[src_lo]);
2140 }
2141 if (dst_lo_rc == rc_stack) {
2142 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2143 } else {
2144 st->print("%s", Matcher::regName[dst_lo]);
2145 }
2146 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2147 int vsize = 0;
2148 switch (ideal_reg()) {
2149 case Op_VecD:
2150 vsize = 64;
2151 break;
2152 case Op_VecX:
2153 vsize = 128;
2154 break;
2155 case Op_VecA:
2156 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2157 break;
2158 default:
2159 assert(false, "bad register type for spill");
2160 ShouldNotReachHere();
2161 }
2162 st->print("\t# vector spill size = %d", vsize);
2163 } else if (ideal_reg() == Op_RegVectMask) {
2164 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2165 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2166 st->print("\t# predicate spill size = %d", vsize);
2167 } else {
2168 st->print("\t# spill size = %d", is64 ? 64 : 32);
2169 }
2170 }
2171
2172 return 0;
2173
2174 }
2175
2176 #ifndef PRODUCT
2177 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2178 if (!ra_)
2179 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2180 else
2181 implementation(nullptr, ra_, false, st);
2182 }
2183 #endif
2184
2185 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2186 implementation(masm, ra_, false, nullptr);
2187 }
2188
2189 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2190 return MachNode::size(ra_);
2191 }
2192
2193 //=============================================================================
2194
2195 #ifndef PRODUCT
2196 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2197 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2198 int reg = ra_->get_reg_first(this);
2199 st->print("add %s, rsp, #%d]\t# box lock",
2200 Matcher::regName[reg], offset);
2201 }
2202 #endif
2203
2204 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2205 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2206 int reg = ra_->get_encode(this);
2207
2208 // This add will handle any 24-bit signed offset. 24 bits allows an
2209 // 8 megabyte stack frame.
2210 __ add(as_Register(reg), sp, offset);
2211 }
2212
2213 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2214 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2215 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2216
2217 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2218 return NativeInstruction::instruction_size;
2219 } else {
2220 return 2 * NativeInstruction::instruction_size;
2221 }
2222 }
2223
2224 //=============================================================================
2225
2226 #ifndef PRODUCT
2227 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2228 {
2229 st->print_cr("# MachUEPNode");
2230 if (UseCompressedClassPointers) {
2231 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2232 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2233 st->print_cr("\tcmpw rscratch1, r10");
2234 } else {
2235 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2236 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2237 st->print_cr("\tcmp rscratch1, r10");
2238 }
2239 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2240 }
2241 #endif
2242
2243 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2244 {
2245 __ ic_check(InteriorEntryAlignment);
2246 }
2247
2248 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2249 {
2250 return MachNode::size(ra_);
2251 }
2252
2253 // REQUIRED EMIT CODE
2254
2255 //=============================================================================
2256
2257 // Emit deopt handler code.
2258 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2259 {
2260 // Note that the code buffer's insts_mark is always relative to insts.
2261 // That's why we must use the macroassembler to generate a handler.
2262 address base = __ start_a_stub(size_deopt_handler());
2263 if (base == nullptr) {
2264 ciEnv::current()->record_failure("CodeCache is full");
2265 return 0; // CodeBuffer::expand failed
2266 }
2267
2268 int offset = __ offset();
2269 Label start;
2270 __ bind(start);
2271 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2272
2273 int entry_offset = __ offset();
2274 __ b(start);
2275
2276 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2277 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2278 "out of bounds read in post-call NOP check");
2279 __ end_a_stub();
2280 return entry_offset;
2281 }
2282
2283 // REQUIRED MATCHER CODE
2284
2285 //=============================================================================
2286
2287 bool Matcher::match_rule_supported(int opcode) {
2288 if (!has_match_rule(opcode))
2289 return false;
2290
2291 switch (opcode) {
2292 case Op_OnSpinWait:
2293 return VM_Version::supports_on_spin_wait();
2294 case Op_CacheWB:
2295 case Op_CacheWBPreSync:
2296 case Op_CacheWBPostSync:
2297 if (!VM_Version::supports_data_cache_line_flush()) {
2298 return false;
2299 }
2300 break;
2301 case Op_ExpandBits:
2302 case Op_CompressBits:
2303 if (!VM_Version::supports_svebitperm()) {
2304 return false;
2305 }
2306 break;
2307 case Op_FmaF:
2308 case Op_FmaD:
2309 case Op_FmaVF:
2310 case Op_FmaVD:
2311 if (!UseFMA) {
2312 return false;
2313 }
2314 break;
2315 case Op_FmaHF:
2316 // UseFMA flag also needs to be checked along with FEAT_FP16
2317 if (!UseFMA || !is_feat_fp16_supported()) {
2318 return false;
2319 }
2320 break;
2321 case Op_AddHF:
2322 case Op_SubHF:
2323 case Op_MulHF:
2324 case Op_DivHF:
2325 case Op_MinHF:
2326 case Op_MaxHF:
2327 case Op_SqrtHF:
2328 // Half-precision floating point scalar operations require FEAT_FP16
2329 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2330 // features are supported.
2331 if (!is_feat_fp16_supported()) {
2332 return false;
2333 }
2334 break;
2335 }
2336
2337 return true; // Per default match rules are supported.
2338 }
2339
2340 const RegMask* Matcher::predicate_reg_mask(void) {
2341 return &_PR_REG_mask;
2342 }
2343
2344 bool Matcher::supports_vector_calling_convention(void) {
2345 return EnableVectorSupport;
2346 }
2347
2348 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2349 assert(EnableVectorSupport, "sanity");
2350 int lo = V0_num;
2351 int hi = V0_H_num;
2352 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2353 hi = V0_K_num;
2354 }
2355 return OptoRegPair(hi, lo);
2356 }
2357
2358 // Is this branch offset short enough that a short branch can be used?
2359 //
2360 // NOTE: If the platform does not provide any short branch variants, then
2361 // this method should return false for offset 0.
2362 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2363 // The passed offset is relative to address of the branch.
2364
2365 return (-32768 <= offset && offset < 32768);
2366 }
2367
2368 // Vector width in bytes.
2369 int Matcher::vector_width_in_bytes(BasicType bt) {
2370 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2371 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2372 // Minimum 2 values in vector
2373 if (size < 2*type2aelembytes(bt)) size = 0;
2374 // But never < 4
2375 if (size < 4) size = 0;
2376 return size;
2377 }
2378
2379 // Limits on vector size (number of elements) loaded into vector.
2380 int Matcher::max_vector_size(const BasicType bt) {
2381 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2382 }
2383
2384 int Matcher::min_vector_size(const BasicType bt) {
2385 // Usually, the shortest vector length supported by AArch64 ISA and
2386 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2387 // vectors in a few special cases.
2388 int size;
2389 switch(bt) {
2390 case T_BOOLEAN:
2391 // Load/store a vector mask with only 2 elements for vector types
2392 // such as "2I/2F/2L/2D".
2393 size = 2;
2394 break;
2395 case T_BYTE:
2396 // Generate a "4B" vector, to support vector cast between "8B/16B"
2397 // and "4S/4I/4L/4F/4D".
2398 size = 4;
2399 break;
2400 case T_SHORT:
2401 // Generate a "2S" vector, to support vector cast between "4S/8S"
2402 // and "2I/2L/2F/2D".
2403 size = 2;
2404 break;
2405 default:
2406 // Limit the min vector length to 64-bit.
2407 size = 8 / type2aelembytes(bt);
2408 // The number of elements in a vector should be at least 2.
2409 size = MAX2(size, 2);
2410 }
2411
2412 int max_size = max_vector_size(bt);
2413 return MIN2(size, max_size);
2414 }
2415
2416 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2417 return Matcher::max_vector_size(bt);
2418 }
2419
2420 // Actual max scalable vector register length.
2421 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2422 return Matcher::max_vector_size(bt);
2423 }
2424
2425 // Vector ideal reg.
2426 uint Matcher::vector_ideal_reg(int len) {
2427 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2428 return Op_VecA;
2429 }
2430 switch(len) {
2431 // For 16-bit/32-bit mask vector, reuse VecD.
2432 case 2:
2433 case 4:
2434 case 8: return Op_VecD;
2435 case 16: return Op_VecX;
2436 }
2437 ShouldNotReachHere();
2438 return 0;
2439 }
2440
2441 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2442 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2443 switch (ideal_reg) {
2444 case Op_VecA: return new vecAOper();
2445 case Op_VecD: return new vecDOper();
2446 case Op_VecX: return new vecXOper();
2447 }
2448 ShouldNotReachHere();
2449 return nullptr;
2450 }
2451
2452 bool Matcher::is_reg2reg_move(MachNode* m) {
2453 return false;
2454 }
2455
2456 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2457 return false;
2458 }
2459
2460 bool Matcher::is_generic_vector(MachOper* opnd) {
2461 return opnd->opcode() == VREG;
2462 }
2463
2464 #ifdef ASSERT
2465 // Return whether or not this register is ever used as an argument.
2466 bool Matcher::can_be_java_arg(int reg)
2467 {
2468 return
2469 reg == R0_num || reg == R0_H_num ||
2470 reg == R1_num || reg == R1_H_num ||
2471 reg == R2_num || reg == R2_H_num ||
2472 reg == R3_num || reg == R3_H_num ||
2473 reg == R4_num || reg == R4_H_num ||
2474 reg == R5_num || reg == R5_H_num ||
2475 reg == R6_num || reg == R6_H_num ||
2476 reg == R7_num || reg == R7_H_num ||
2477 reg == V0_num || reg == V0_H_num ||
2478 reg == V1_num || reg == V1_H_num ||
2479 reg == V2_num || reg == V2_H_num ||
2480 reg == V3_num || reg == V3_H_num ||
2481 reg == V4_num || reg == V4_H_num ||
2482 reg == V5_num || reg == V5_H_num ||
2483 reg == V6_num || reg == V6_H_num ||
2484 reg == V7_num || reg == V7_H_num;
2485 }
2486 #endif
2487
2488 uint Matcher::int_pressure_limit()
2489 {
2490 // JDK-8183543: When taking the number of available registers as int
2491 // register pressure threshold, the jtreg test:
2492 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2493 // failed due to C2 compilation failure with
2494 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2495 //
2496 // A derived pointer is live at CallNode and then is flagged by RA
2497 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2498 // derived pointers and lastly fail to spill after reaching maximum
2499 // number of iterations. Lowering the default pressure threshold to
2500 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2501 // a high register pressure area of the code so that split_DEF can
2502 // generate DefinitionSpillCopy for the derived pointer.
2503 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2504 if (!PreserveFramePointer) {
2505 // When PreserveFramePointer is off, frame pointer is allocatable,
2506 // but different from other SOC registers, it is excluded from
2507 // fatproj's mask because its save type is No-Save. Decrease 1 to
2508 // ensure high pressure at fatproj when PreserveFramePointer is off.
2509 // See check_pressure_at_fatproj().
2510 default_int_pressure_threshold--;
2511 }
2512 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2513 }
2514
2515 uint Matcher::float_pressure_limit()
2516 {
2517 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2518 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2519 }
2520
2521 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2522 return false;
2523 }
2524
2525 const RegMask& Matcher::divI_proj_mask() {
2526 ShouldNotReachHere();
2527 return RegMask::EMPTY;
2528 }
2529
2530 // Register for MODI projection of divmodI.
2531 const RegMask& Matcher::modI_proj_mask() {
2532 ShouldNotReachHere();
2533 return RegMask::EMPTY;
2534 }
2535
2536 // Register for DIVL projection of divmodL.
2537 const RegMask& Matcher::divL_proj_mask() {
2538 ShouldNotReachHere();
2539 return RegMask::EMPTY;
2540 }
2541
2542 // Register for MODL projection of divmodL.
2543 const RegMask& Matcher::modL_proj_mask() {
2544 ShouldNotReachHere();
2545 return RegMask::EMPTY;
2546 }
2547
2548 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2549 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2550 Node* u = addp->fast_out(i);
2551 if (u->is_LoadStore()) {
2552 // On AArch64, LoadStoreNodes (i.e. compare and swap
2553 // instructions) only take register indirect as an operand, so
2554 // any attempt to use an AddPNode as an input to a LoadStoreNode
2555 // must fail.
2556 return false;
2557 }
2558 if (u->is_Mem()) {
2559 int opsize = u->as_Mem()->memory_size();
2560 assert(opsize > 0, "unexpected memory operand size");
2561 if (u->as_Mem()->memory_size() != (1<<shift)) {
2562 return false;
2563 }
2564 }
2565 }
2566 return true;
2567 }
2568
2569 // Convert BoolTest condition to Assembler condition.
2570 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2571 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2572 Assembler::Condition result;
2573 switch(cond) {
2574 case BoolTest::eq:
2575 result = Assembler::EQ; break;
2576 case BoolTest::ne:
2577 result = Assembler::NE; break;
2578 case BoolTest::le:
2579 result = Assembler::LE; break;
2580 case BoolTest::ge:
2581 result = Assembler::GE; break;
2582 case BoolTest::lt:
2583 result = Assembler::LT; break;
2584 case BoolTest::gt:
2585 result = Assembler::GT; break;
2586 case BoolTest::ule:
2587 result = Assembler::LS; break;
2588 case BoolTest::uge:
2589 result = Assembler::HS; break;
2590 case BoolTest::ult:
2591 result = Assembler::LO; break;
2592 case BoolTest::ugt:
2593 result = Assembler::HI; break;
2594 case BoolTest::overflow:
2595 result = Assembler::VS; break;
2596 case BoolTest::no_overflow:
2597 result = Assembler::VC; break;
2598 default:
2599 ShouldNotReachHere();
2600 return Assembler::Condition(-1);
2601 }
2602
2603 // Check conversion
2604 if (cond & BoolTest::unsigned_compare) {
2605 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2606 } else {
2607 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2608 }
2609
2610 return result;
2611 }
2612
2613 // Binary src (Replicate con)
2614 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2615 if (n == nullptr || m == nullptr) {
2616 return false;
2617 }
2618
2619 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2620 return false;
2621 }
2622
2623 Node* imm_node = m->in(1);
2624 if (!imm_node->is_Con()) {
2625 return false;
2626 }
2627
2628 const Type* t = imm_node->bottom_type();
2629 if (!(t->isa_int() || t->isa_long())) {
2630 return false;
2631 }
2632
2633 switch (n->Opcode()) {
2634 case Op_AndV:
2635 case Op_OrV:
2636 case Op_XorV: {
2637 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2638 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2639 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2640 }
2641 case Op_AddVB:
2642 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2643 case Op_AddVS:
2644 case Op_AddVI:
2645 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2646 case Op_AddVL:
2647 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2648 default:
2649 return false;
2650 }
2651 }
2652
2653 // (XorV src (Replicate m1))
2654 // (XorVMask src (MaskAll m1))
2655 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2656 if (n != nullptr && m != nullptr) {
2657 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2658 VectorNode::is_all_ones_vector(m);
2659 }
2660 return false;
2661 }
2662
2663 // Should the matcher clone input 'm' of node 'n'?
2664 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2665 if (is_vshift_con_pattern(n, m) ||
2666 is_vector_bitwise_not_pattern(n, m) ||
2667 is_valid_sve_arith_imm_pattern(n, m) ||
2668 is_encode_and_store_pattern(n, m)) {
2669 mstack.push(m, Visit);
2670 return true;
2671 }
2672 return false;
2673 }
2674
2675 // Should the Matcher clone shifts on addressing modes, expecting them
2676 // to be subsumed into complex addressing expressions or compute them
2677 // into registers?
2678 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2679
2680 // Loads and stores with indirect memory input (e.g., volatile loads and
2681 // stores) do not subsume the input into complex addressing expressions. If
2682 // the addressing expression is input to at least one such load or store, do
2683 // not clone the addressing expression. Query needs_acquiring_load and
2684 // needs_releasing_store as a proxy for indirect memory input, as it is not
2685 // possible to directly query for indirect memory input at this stage.
2686 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2687 Node* n = m->fast_out(i);
2688 if (n->is_Load() && needs_acquiring_load(n)) {
2689 return false;
2690 }
2691 if (n->is_Store() && needs_releasing_store(n)) {
2692 return false;
2693 }
2694 }
2695
2696 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2697 return true;
2698 }
2699
2700 Node *off = m->in(AddPNode::Offset);
2701 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2702 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2703 // Are there other uses besides address expressions?
2704 !is_visited(off)) {
2705 address_visited.set(off->_idx); // Flag as address_visited
2706 mstack.push(off->in(2), Visit);
2707 Node *conv = off->in(1);
2708 if (conv->Opcode() == Op_ConvI2L &&
2709 // Are there other uses besides address expressions?
2710 !is_visited(conv)) {
2711 address_visited.set(conv->_idx); // Flag as address_visited
2712 mstack.push(conv->in(1), Pre_Visit);
2713 } else {
2714 mstack.push(conv, Pre_Visit);
2715 }
2716 address_visited.test_set(m->_idx); // Flag as address_visited
2717 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2718 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2719 return true;
2720 } else if (off->Opcode() == Op_ConvI2L &&
2721 // Are there other uses besides address expressions?
2722 !is_visited(off)) {
2723 address_visited.test_set(m->_idx); // Flag as address_visited
2724 address_visited.set(off->_idx); // Flag as address_visited
2725 mstack.push(off->in(1), Pre_Visit);
2726 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2727 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2728 return true;
2729 }
2730 return false;
2731 }
2732
2733 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2734 { \
2735 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2736 guarantee(DISP == 0, "mode not permitted for volatile"); \
2737 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2738 __ INSN(REG, as_Register(BASE)); \
2739 }
2740
2741
2742 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2743 {
2744 Address::extend scale;
2745
2746 // Hooboy, this is fugly. We need a way to communicate to the
2747 // encoder that the index needs to be sign extended, so we have to
2748 // enumerate all the cases.
2749 switch (opcode) {
2750 case INDINDEXSCALEDI2L:
2751 case INDINDEXSCALEDI2LN:
2752 case INDINDEXI2L:
2753 case INDINDEXI2LN:
2754 scale = Address::sxtw(size);
2755 break;
2756 default:
2757 scale = Address::lsl(size);
2758 }
2759
2760 if (index == -1) {
2761 return Address(base, disp);
2762 } else {
2763 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2764 return Address(base, as_Register(index), scale);
2765 }
2766 }
2767
2768
2769 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2770 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2771 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2772 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2773 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2774
2775 // Used for all non-volatile memory accesses. The use of
2776 // $mem->opcode() to discover whether this pattern uses sign-extended
2777 // offsets is something of a kludge.
2778 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2779 Register reg, int opcode,
2780 Register base, int index, int scale, int disp,
2781 int size_in_memory)
2782 {
2783 Address addr = mem2address(opcode, base, index, scale, disp);
2784 if (addr.getMode() == Address::base_plus_offset) {
2785 /* Fix up any out-of-range offsets. */
2786 assert_different_registers(rscratch1, base);
2787 assert_different_registers(rscratch1, reg);
2788 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2789 }
2790 (masm->*insn)(reg, addr);
2791 }
2792
2793 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2794 FloatRegister reg, int opcode,
2795 Register base, int index, int size, int disp,
2796 int size_in_memory)
2797 {
2798 Address::extend scale;
2799
2800 switch (opcode) {
2801 case INDINDEXSCALEDI2L:
2802 case INDINDEXSCALEDI2LN:
2803 scale = Address::sxtw(size);
2804 break;
2805 default:
2806 scale = Address::lsl(size);
2807 }
2808
2809 if (index == -1) {
2810 // Fix up any out-of-range offsets.
2811 assert_different_registers(rscratch1, base);
2812 Address addr = Address(base, disp);
2813 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2814 (masm->*insn)(reg, addr);
2815 } else {
2816 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2817 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2818 }
2819 }
2820
2821 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2822 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2823 int opcode, Register base, int index, int size, int disp)
2824 {
2825 if (index == -1) {
2826 (masm->*insn)(reg, T, Address(base, disp));
2827 } else {
2828 assert(disp == 0, "unsupported address mode");
2829 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2830 }
2831 }
2832
2833 %}
2834
2835
2836
2837 //----------ENCODING BLOCK-----------------------------------------------------
2838 // This block specifies the encoding classes used by the compiler to
2839 // output byte streams. Encoding classes are parameterized macros
2840 // used by Machine Instruction Nodes in order to generate the bit
2841 // encoding of the instruction. Operands specify their base encoding
2842 // interface with the interface keyword. There are currently
2843 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2844 // COND_INTER. REG_INTER causes an operand to generate a function
2845 // which returns its register number when queried. CONST_INTER causes
2846 // an operand to generate a function which returns the value of the
2847 // constant when queried. MEMORY_INTER causes an operand to generate
2848 // four functions which return the Base Register, the Index Register,
2849 // the Scale Value, and the Offset Value of the operand when queried.
2850 // COND_INTER causes an operand to generate six functions which return
2851 // the encoding code (ie - encoding bits for the instruction)
2852 // associated with each basic boolean condition for a conditional
2853 // instruction.
2854 //
2855 // Instructions specify two basic values for encoding. Again, a
2856 // function is available to check if the constant displacement is an
2857 // oop. They use the ins_encode keyword to specify their encoding
2858 // classes (which must be a sequence of enc_class names, and their
2859 // parameters, specified in the encoding block), and they use the
2860 // opcode keyword to specify, in order, their primary, secondary, and
2861 // tertiary opcode. Only the opcode sections which a particular
2862 // instruction needs for encoding need to be specified.
2863 encode %{
2864 // Build emit functions for each basic byte or larger field in the
2865 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2866 // from C++ code in the enc_class source block. Emit functions will
2867 // live in the main source block for now. In future, we can
2868 // generalize this by adding a syntax that specifies the sizes of
2869 // fields in an order, so that the adlc can build the emit functions
2870 // automagically
2871
2872 // catch all for unimplemented encodings
2873 enc_class enc_unimplemented %{
2874 __ unimplemented("C2 catch all");
2875 %}
2876
2877 // BEGIN Non-volatile memory access
2878
2879 // This encoding class is generated automatically from ad_encode.m4.
2880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2881 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2882 Register dst_reg = as_Register($dst$$reg);
2883 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2884 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2885 %}
2886
2887 // This encoding class is generated automatically from ad_encode.m4.
2888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2889 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2890 Register dst_reg = as_Register($dst$$reg);
2891 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2892 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2893 %}
2894
2895 // This encoding class is generated automatically from ad_encode.m4.
2896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2897 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2898 Register dst_reg = as_Register($dst$$reg);
2899 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2900 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2901 %}
2902
2903 // This encoding class is generated automatically from ad_encode.m4.
2904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2905 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2906 Register dst_reg = as_Register($dst$$reg);
2907 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2908 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2909 %}
2910
2911 // This encoding class is generated automatically from ad_encode.m4.
2912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2913 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2914 Register dst_reg = as_Register($dst$$reg);
2915 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2916 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2917 %}
2918
2919 // This encoding class is generated automatically from ad_encode.m4.
2920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2921 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2922 Register dst_reg = as_Register($dst$$reg);
2923 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2924 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2925 %}
2926
2927 // This encoding class is generated automatically from ad_encode.m4.
2928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2929 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2930 Register dst_reg = as_Register($dst$$reg);
2931 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2932 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2933 %}
2934
2935 // This encoding class is generated automatically from ad_encode.m4.
2936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2937 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2938 Register dst_reg = as_Register($dst$$reg);
2939 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2940 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2941 %}
2942
2943 // This encoding class is generated automatically from ad_encode.m4.
2944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2945 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2946 Register dst_reg = as_Register($dst$$reg);
2947 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2948 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2949 %}
2950
2951 // This encoding class is generated automatically from ad_encode.m4.
2952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2953 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2954 Register dst_reg = as_Register($dst$$reg);
2955 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2956 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2957 %}
2958
2959 // This encoding class is generated automatically from ad_encode.m4.
2960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2961 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2962 Register dst_reg = as_Register($dst$$reg);
2963 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2964 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2965 %}
2966
2967 // This encoding class is generated automatically from ad_encode.m4.
2968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2969 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2970 Register dst_reg = as_Register($dst$$reg);
2971 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2972 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2973 %}
2974
2975 // This encoding class is generated automatically from ad_encode.m4.
2976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2977 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2978 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2979 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2980 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2981 %}
2982
2983 // This encoding class is generated automatically from ad_encode.m4.
2984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2985 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2986 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2987 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2988 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2989 %}
2990
2991 // This encoding class is generated automatically from ad_encode.m4.
2992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2993 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2994 Register src_reg = as_Register($src$$reg);
2995 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2996 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2997 %}
2998
2999 // This encoding class is generated automatically from ad_encode.m4.
3000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3001 enc_class aarch64_enc_strb0(memory1 mem) %{
3002 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3003 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3004 %}
3005
3006 // This encoding class is generated automatically from ad_encode.m4.
3007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3008 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3009 Register src_reg = as_Register($src$$reg);
3010 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3011 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3012 %}
3013
3014 // This encoding class is generated automatically from ad_encode.m4.
3015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3016 enc_class aarch64_enc_strh0(memory2 mem) %{
3017 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3018 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3019 %}
3020
3021 // This encoding class is generated automatically from ad_encode.m4.
3022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3023 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3024 Register src_reg = as_Register($src$$reg);
3025 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3026 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3027 %}
3028
3029 // This encoding class is generated automatically from ad_encode.m4.
3030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3031 enc_class aarch64_enc_strw0(memory4 mem) %{
3032 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3033 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3034 %}
3035
3036 // This encoding class is generated automatically from ad_encode.m4.
3037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3038 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3039 Register src_reg = as_Register($src$$reg);
3040 // we sometimes get asked to store the stack pointer into the
3041 // current thread -- we cannot do that directly on AArch64
3042 if (src_reg == r31_sp) {
3043 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3044 __ mov(rscratch2, sp);
3045 src_reg = rscratch2;
3046 }
3047 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3048 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3049 %}
3050
3051 // This encoding class is generated automatically from ad_encode.m4.
3052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3053 enc_class aarch64_enc_str0(memory8 mem) %{
3054 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3055 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3056 %}
3057
3058 // This encoding class is generated automatically from ad_encode.m4.
3059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3060 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3061 FloatRegister src_reg = as_FloatRegister($src$$reg);
3062 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3063 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3064 %}
3065
3066 // This encoding class is generated automatically from ad_encode.m4.
3067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3068 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3069 FloatRegister src_reg = as_FloatRegister($src$$reg);
3070 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3071 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3072 %}
3073
3074 // This encoding class is generated automatically from ad_encode.m4.
3075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3076 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3077 __ membar(Assembler::StoreStore);
3078 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3079 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3080 %}
3081
3082 // END Non-volatile memory access
3083
3084 // Vector loads and stores
3085 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3086 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3087 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3088 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3089 %}
3090
3091 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3092 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3093 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3094 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3095 %}
3096
3097 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3098 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3099 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3100 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3101 %}
3102
3103 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3104 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3105 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3106 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3107 %}
3108
3109 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3110 FloatRegister src_reg = as_FloatRegister($src$$reg);
3111 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3112 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3113 %}
3114
3115 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3116 FloatRegister src_reg = as_FloatRegister($src$$reg);
3117 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3118 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3119 %}
3120
3121 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3122 FloatRegister src_reg = as_FloatRegister($src$$reg);
3123 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3124 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3125 %}
3126
3127 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3128 FloatRegister src_reg = as_FloatRegister($src$$reg);
3129 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3130 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3131 %}
3132
3133 // volatile loads and stores
3134
3135 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3136 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3137 rscratch1, stlrb);
3138 %}
3139
3140 enc_class aarch64_enc_stlrb0(memory mem) %{
3141 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, stlrb);
3143 %}
3144
3145 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3146 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3147 rscratch1, stlrh);
3148 %}
3149
3150 enc_class aarch64_enc_stlrh0(memory mem) %{
3151 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3152 rscratch1, stlrh);
3153 %}
3154
3155 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3156 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3157 rscratch1, stlrw);
3158 %}
3159
3160 enc_class aarch64_enc_stlrw0(memory mem) %{
3161 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3162 rscratch1, stlrw);
3163 %}
3164
3165 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3166 Register dst_reg = as_Register($dst$$reg);
3167 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3168 rscratch1, ldarb);
3169 __ sxtbw(dst_reg, dst_reg);
3170 %}
3171
3172 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3173 Register dst_reg = as_Register($dst$$reg);
3174 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3175 rscratch1, ldarb);
3176 __ sxtb(dst_reg, dst_reg);
3177 %}
3178
3179 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3180 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldarb);
3182 %}
3183
3184 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3185 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3186 rscratch1, ldarb);
3187 %}
3188
3189 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3190 Register dst_reg = as_Register($dst$$reg);
3191 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3192 rscratch1, ldarh);
3193 __ sxthw(dst_reg, dst_reg);
3194 %}
3195
3196 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3197 Register dst_reg = as_Register($dst$$reg);
3198 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3199 rscratch1, ldarh);
3200 __ sxth(dst_reg, dst_reg);
3201 %}
3202
3203 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3204 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3205 rscratch1, ldarh);
3206 %}
3207
3208 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3209 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3210 rscratch1, ldarh);
3211 %}
3212
3213 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3214 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3215 rscratch1, ldarw);
3216 %}
3217
3218 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3219 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3220 rscratch1, ldarw);
3221 %}
3222
3223 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3224 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3225 rscratch1, ldar);
3226 %}
3227
3228 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3229 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3230 rscratch1, ldarw);
3231 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3232 %}
3233
3234 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3235 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3236 rscratch1, ldar);
3237 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3238 %}
3239
3240 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3241 Register src_reg = as_Register($src$$reg);
3242 // we sometimes get asked to store the stack pointer into the
3243 // current thread -- we cannot do that directly on AArch64
3244 if (src_reg == r31_sp) {
3245 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3246 __ mov(rscratch2, sp);
3247 src_reg = rscratch2;
3248 }
3249 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3250 rscratch1, stlr);
3251 %}
3252
3253 enc_class aarch64_enc_stlr0(memory mem) %{
3254 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3255 rscratch1, stlr);
3256 %}
3257
3258 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3259 {
3260 FloatRegister src_reg = as_FloatRegister($src$$reg);
3261 __ fmovs(rscratch2, src_reg);
3262 }
3263 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3264 rscratch1, stlrw);
3265 %}
3266
3267 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3268 {
3269 FloatRegister src_reg = as_FloatRegister($src$$reg);
3270 __ fmovd(rscratch2, src_reg);
3271 }
3272 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3273 rscratch1, stlr);
3274 %}
3275
3276 // synchronized read/update encodings
3277
3278 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3279 Register dst_reg = as_Register($dst$$reg);
3280 Register base = as_Register($mem$$base);
3281 int index = $mem$$index;
3282 int scale = $mem$$scale;
3283 int disp = $mem$$disp;
3284 if (index == -1) {
3285 if (disp != 0) {
3286 __ lea(rscratch1, Address(base, disp));
3287 __ ldaxr(dst_reg, rscratch1);
3288 } else {
3289 // TODO
3290 // should we ever get anything other than this case?
3291 __ ldaxr(dst_reg, base);
3292 }
3293 } else {
3294 Register index_reg = as_Register(index);
3295 if (disp == 0) {
3296 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3297 __ ldaxr(dst_reg, rscratch1);
3298 } else {
3299 __ lea(rscratch1, Address(base, disp));
3300 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3301 __ ldaxr(dst_reg, rscratch1);
3302 }
3303 }
3304 %}
3305
3306 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3307 Register src_reg = as_Register($src$$reg);
3308 Register base = as_Register($mem$$base);
3309 int index = $mem$$index;
3310 int scale = $mem$$scale;
3311 int disp = $mem$$disp;
3312 if (index == -1) {
3313 if (disp != 0) {
3314 __ lea(rscratch2, Address(base, disp));
3315 __ stlxr(rscratch1, src_reg, rscratch2);
3316 } else {
3317 // TODO
3318 // should we ever get anything other than this case?
3319 __ stlxr(rscratch1, src_reg, base);
3320 }
3321 } else {
3322 Register index_reg = as_Register(index);
3323 if (disp == 0) {
3324 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3325 __ stlxr(rscratch1, src_reg, rscratch2);
3326 } else {
3327 __ lea(rscratch2, Address(base, disp));
3328 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3329 __ stlxr(rscratch1, src_reg, rscratch2);
3330 }
3331 }
3332 __ cmpw(rscratch1, zr);
3333 %}
3334
3335 // prefetch encodings
3336
3337 enc_class aarch64_enc_prefetchw(memory mem) %{
3338 Register base = as_Register($mem$$base);
3339 int index = $mem$$index;
3340 int scale = $mem$$scale;
3341 int disp = $mem$$disp;
3342 if (index == -1) {
3343 // Fix up any out-of-range offsets.
3344 assert_different_registers(rscratch1, base);
3345 Address addr = Address(base, disp);
3346 addr = __ legitimize_address(addr, 8, rscratch1);
3347 __ prfm(addr, PSTL1KEEP);
3348 } else {
3349 Register index_reg = as_Register(index);
3350 if (disp == 0) {
3351 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3352 } else {
3353 __ lea(rscratch1, Address(base, disp));
3354 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3355 }
3356 }
3357 %}
3358
3359 // mov encodings
3360
3361 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3362 uint32_t con = (uint32_t)$src$$constant;
3363 Register dst_reg = as_Register($dst$$reg);
3364 if (con == 0) {
3365 __ movw(dst_reg, zr);
3366 } else {
3367 __ movw(dst_reg, con);
3368 }
3369 %}
3370
3371 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3372 Register dst_reg = as_Register($dst$$reg);
3373 uint64_t con = (uint64_t)$src$$constant;
3374 if (con == 0) {
3375 __ mov(dst_reg, zr);
3376 } else {
3377 __ mov(dst_reg, con);
3378 }
3379 %}
3380
3381 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3382 Register dst_reg = as_Register($dst$$reg);
3383 address con = (address)$src$$constant;
3384 if (con == nullptr || con == (address)1) {
3385 ShouldNotReachHere();
3386 } else {
3387 relocInfo::relocType rtype = $src->constant_reloc();
3388 if (rtype == relocInfo::oop_type) {
3389 __ movoop(dst_reg, (jobject)con);
3390 } else if (rtype == relocInfo::metadata_type) {
3391 __ mov_metadata(dst_reg, (Metadata*)con);
3392 } else {
3393 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3394 // load fake address constants using a normal move
3395 if (! __ is_valid_AArch64_address(con) ||
3396 con < (address)(uintptr_t)os::vm_page_size()) {
3397 __ mov(dst_reg, con);
3398 } else {
3399 // no reloc so just use adrp and add
3400 uint64_t offset;
3401 __ adrp(dst_reg, con, offset);
3402 __ add(dst_reg, dst_reg, offset);
3403 }
3404 }
3405 }
3406 %}
3407
3408 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3409 Register dst_reg = as_Register($dst$$reg);
3410 __ mov(dst_reg, zr);
3411 %}
3412
3413 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3414 Register dst_reg = as_Register($dst$$reg);
3415 __ mov(dst_reg, (uint64_t)1);
3416 %}
3417
3418 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3419 Register dst_reg = as_Register($dst$$reg);
3420 address con = (address)$src$$constant;
3421 if (con == nullptr) {
3422 ShouldNotReachHere();
3423 } else {
3424 relocInfo::relocType rtype = $src->constant_reloc();
3425 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3426 __ set_narrow_oop(dst_reg, (jobject)con);
3427 }
3428 %}
3429
3430 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3431 Register dst_reg = as_Register($dst$$reg);
3432 __ mov(dst_reg, zr);
3433 %}
3434
3435 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3436 Register dst_reg = as_Register($dst$$reg);
3437 address con = (address)$src$$constant;
3438 if (con == nullptr) {
3439 ShouldNotReachHere();
3440 } else {
3441 relocInfo::relocType rtype = $src->constant_reloc();
3442 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3443 __ set_narrow_klass(dst_reg, (Klass *)con);
3444 }
3445 %}
3446
3447 // arithmetic encodings
3448
3449 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3450 Register dst_reg = as_Register($dst$$reg);
3451 Register src_reg = as_Register($src1$$reg);
3452 int32_t con = (int32_t)$src2$$constant;
3453 // add has primary == 0, subtract has primary == 1
3454 if ($primary) { con = -con; }
3455 if (con < 0) {
3456 __ subw(dst_reg, src_reg, -con);
3457 } else {
3458 __ addw(dst_reg, src_reg, con);
3459 }
3460 %}
3461
3462 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3463 Register dst_reg = as_Register($dst$$reg);
3464 Register src_reg = as_Register($src1$$reg);
3465 int32_t con = (int32_t)$src2$$constant;
3466 // add has primary == 0, subtract has primary == 1
3467 if ($primary) { con = -con; }
3468 if (con < 0) {
3469 __ sub(dst_reg, src_reg, -con);
3470 } else {
3471 __ add(dst_reg, src_reg, con);
3472 }
3473 %}
3474
3475 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3476 Register dst_reg = as_Register($dst$$reg);
3477 Register src1_reg = as_Register($src1$$reg);
3478 Register src2_reg = as_Register($src2$$reg);
3479 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3480 %}
3481
3482 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3483 Register dst_reg = as_Register($dst$$reg);
3484 Register src1_reg = as_Register($src1$$reg);
3485 Register src2_reg = as_Register($src2$$reg);
3486 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3487 %}
3488
3489 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3490 Register dst_reg = as_Register($dst$$reg);
3491 Register src1_reg = as_Register($src1$$reg);
3492 Register src2_reg = as_Register($src2$$reg);
3493 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3494 %}
3495
3496 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3497 Register dst_reg = as_Register($dst$$reg);
3498 Register src1_reg = as_Register($src1$$reg);
3499 Register src2_reg = as_Register($src2$$reg);
3500 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3501 %}
3502
3503 // compare instruction encodings
3504
3505 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3506 Register reg1 = as_Register($src1$$reg);
3507 Register reg2 = as_Register($src2$$reg);
3508 __ cmpw(reg1, reg2);
3509 %}
3510
3511 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3512 Register reg = as_Register($src1$$reg);
3513 int32_t val = $src2$$constant;
3514 if (val >= 0) {
3515 __ subsw(zr, reg, val);
3516 } else {
3517 __ addsw(zr, reg, -val);
3518 }
3519 %}
3520
3521 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3522 Register reg1 = as_Register($src1$$reg);
3523 uint32_t val = (uint32_t)$src2$$constant;
3524 __ movw(rscratch1, val);
3525 __ cmpw(reg1, rscratch1);
3526 %}
3527
3528 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3529 Register reg1 = as_Register($src1$$reg);
3530 Register reg2 = as_Register($src2$$reg);
3531 __ cmp(reg1, reg2);
3532 %}
3533
3534 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3535 Register reg = as_Register($src1$$reg);
3536 int64_t val = $src2$$constant;
3537 if (val >= 0) {
3538 __ subs(zr, reg, val);
3539 } else if (val != -val) {
3540 __ adds(zr, reg, -val);
3541 } else {
3542 // aargh, Long.MIN_VALUE is a special case
3543 __ orr(rscratch1, zr, (uint64_t)val);
3544 __ subs(zr, reg, rscratch1);
3545 }
3546 %}
3547
3548 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3549 Register reg1 = as_Register($src1$$reg);
3550 uint64_t val = (uint64_t)$src2$$constant;
3551 __ mov(rscratch1, val);
3552 __ cmp(reg1, rscratch1);
3553 %}
3554
3555 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3556 Register reg1 = as_Register($src1$$reg);
3557 Register reg2 = as_Register($src2$$reg);
3558 __ cmp(reg1, reg2);
3559 %}
3560
3561 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3562 Register reg1 = as_Register($src1$$reg);
3563 Register reg2 = as_Register($src2$$reg);
3564 __ cmpw(reg1, reg2);
3565 %}
3566
3567 enc_class aarch64_enc_testp(iRegP src) %{
3568 Register reg = as_Register($src$$reg);
3569 __ cmp(reg, zr);
3570 %}
3571
3572 enc_class aarch64_enc_testn(iRegN src) %{
3573 Register reg = as_Register($src$$reg);
3574 __ cmpw(reg, zr);
3575 %}
3576
3577 enc_class aarch64_enc_b(label lbl) %{
3578 Label *L = $lbl$$label;
3579 __ b(*L);
3580 %}
3581
3582 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3583 Label *L = $lbl$$label;
3584 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3585 %}
3586
3587 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3588 Label *L = $lbl$$label;
3589 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3590 %}
3591
3592 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3593 %{
3594 Register sub_reg = as_Register($sub$$reg);
3595 Register super_reg = as_Register($super$$reg);
3596 Register temp_reg = as_Register($temp$$reg);
3597 Register result_reg = as_Register($result$$reg);
3598
3599 Label miss;
3600 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3601 nullptr, &miss,
3602 /*set_cond_codes:*/ true);
3603 if ($primary) {
3604 __ mov(result_reg, zr);
3605 }
3606 __ bind(miss);
3607 %}
3608
3609 enc_class aarch64_enc_java_static_call(method meth) %{
3610 address addr = (address)$meth$$method;
3611 address call;
3612 if (!_method) {
3613 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3614 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3615 if (call == nullptr) {
3616 ciEnv::current()->record_failure("CodeCache is full");
3617 return;
3618 }
3619 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3620 // The NOP here is purely to ensure that eliding a call to
3621 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3622 __ nop();
3623 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3624 } else {
3625 int method_index = resolved_method_index(masm);
3626 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3627 : static_call_Relocation::spec(method_index);
3628 call = __ trampoline_call(Address(addr, rspec));
3629 if (call == nullptr) {
3630 ciEnv::current()->record_failure("CodeCache is full");
3631 return;
3632 }
3633 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3634 // Calls of the same statically bound method can share
3635 // a stub to the interpreter.
3636 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3637 } else {
3638 // Emit stub for static call
3639 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3640 if (stub == nullptr) {
3641 ciEnv::current()->record_failure("CodeCache is full");
3642 return;
3643 }
3644 }
3645 }
3646
3647 __ post_call_nop();
3648
3649 // Only non uncommon_trap calls need to reinitialize ptrue.
3650 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3651 __ reinitialize_ptrue();
3652 }
3653 %}
3654
3655 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3656 int method_index = resolved_method_index(masm);
3657 address call = __ ic_call((address)$meth$$method, method_index);
3658 if (call == nullptr) {
3659 ciEnv::current()->record_failure("CodeCache is full");
3660 return;
3661 }
3662 __ post_call_nop();
3663 if (Compile::current()->max_vector_size() > 0) {
3664 __ reinitialize_ptrue();
3665 }
3666 %}
3667
3668 enc_class aarch64_enc_call_epilog() %{
3669 if (VerifyStackAtCalls) {
3670 // Check that stack depth is unchanged: find majik cookie on stack
3671 __ call_Unimplemented();
3672 }
3673 %}
3674
3675 enc_class aarch64_enc_java_to_runtime(method meth) %{
3676 // some calls to generated routines (arraycopy code) are scheduled
3677 // by C2 as runtime calls. if so we can call them using a br (they
3678 // will be in a reachable segment) otherwise we have to use a blr
3679 // which loads the absolute address into a register.
3680 address entry = (address)$meth$$method;
3681 CodeBlob *cb = CodeCache::find_blob(entry);
3682 if (cb) {
3683 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3684 if (call == nullptr) {
3685 ciEnv::current()->record_failure("CodeCache is full");
3686 return;
3687 }
3688 __ post_call_nop();
3689 } else {
3690 Label retaddr;
3691 // Make the anchor frame walkable
3692 __ adr(rscratch2, retaddr);
3693 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3694 __ lea(rscratch1, RuntimeAddress(entry));
3695 __ blr(rscratch1);
3696 __ bind(retaddr);
3697 __ post_call_nop();
3698 }
3699 if (Compile::current()->max_vector_size() > 0) {
3700 __ reinitialize_ptrue();
3701 }
3702 %}
3703
3704 enc_class aarch64_enc_rethrow() %{
3705 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3706 %}
3707
3708 enc_class aarch64_enc_ret() %{
3709 #ifdef ASSERT
3710 if (Compile::current()->max_vector_size() > 0) {
3711 __ verify_ptrue();
3712 }
3713 #endif
3714 __ ret(lr);
3715 %}
3716
3717 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3718 Register target_reg = as_Register($jump_target$$reg);
3719 __ br(target_reg);
3720 %}
3721
3722 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3723 Register target_reg = as_Register($jump_target$$reg);
3724 // exception oop should be in r0
3725 // ret addr has been popped into lr
3726 // callee expects it in r3
3727 __ mov(r3, lr);
3728 __ br(target_reg);
3729 %}
3730
3731 %}
3732
3733 //----------FRAME--------------------------------------------------------------
3734 // Definition of frame structure and management information.
3735 //
3736 // S T A C K L A Y O U T Allocators stack-slot number
3737 // | (to get allocators register number
3738 // G Owned by | | v add OptoReg::stack0())
3739 // r CALLER | |
3740 // o | +--------+ pad to even-align allocators stack-slot
3741 // w V | pad0 | numbers; owned by CALLER
3742 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3743 // h ^ | in | 5
3744 // | | args | 4 Holes in incoming args owned by SELF
3745 // | | | | 3
3746 // | | +--------+
3747 // V | | old out| Empty on Intel, window on Sparc
3748 // | old |preserve| Must be even aligned.
3749 // | SP-+--------+----> Matcher::_old_SP, even aligned
3750 // | | in | 3 area for Intel ret address
3751 // Owned by |preserve| Empty on Sparc.
3752 // SELF +--------+
3753 // | | pad2 | 2 pad to align old SP
3754 // | +--------+ 1
3755 // | | locks | 0
3756 // | +--------+----> OptoReg::stack0(), even aligned
3757 // | | pad1 | 11 pad to align new SP
3758 // | +--------+
3759 // | | | 10
3760 // | | spills | 9 spills
3761 // V | | 8 (pad0 slot for callee)
3762 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3763 // ^ | out | 7
3764 // | | args | 6 Holes in outgoing args owned by CALLEE
3765 // Owned by +--------+
3766 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3767 // | new |preserve| Must be even-aligned.
3768 // | SP-+--------+----> Matcher::_new_SP, even aligned
3769 // | | |
3770 //
3771 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3772 // known from SELF's arguments and the Java calling convention.
3773 // Region 6-7 is determined per call site.
3774 // Note 2: If the calling convention leaves holes in the incoming argument
3775 // area, those holes are owned by SELF. Holes in the outgoing area
3776 // are owned by the CALLEE. Holes should not be necessary in the
3777 // incoming area, as the Java calling convention is completely under
3778 // the control of the AD file. Doubles can be sorted and packed to
3779 // avoid holes. Holes in the outgoing arguments may be necessary for
3780 // varargs C calling conventions.
3781 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3782 // even aligned with pad0 as needed.
3783 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3784 // (the latter is true on Intel but is it false on AArch64?)
3785 // region 6-11 is even aligned; it may be padded out more so that
3786 // the region from SP to FP meets the minimum stack alignment.
3787 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3788 // alignment. Region 11, pad1, may be dynamically extended so that
3789 // SP meets the minimum alignment.
3790
3791 frame %{
3792 // These three registers define part of the calling convention
3793 // between compiled code and the interpreter.
3794
3795 // Inline Cache Register or Method for I2C.
3796 inline_cache_reg(R12);
3797
3798 // Number of stack slots consumed by locking an object
3799 sync_stack_slots(2);
3800
3801 // Compiled code's Frame Pointer
3802 frame_pointer(R31);
3803
3804 // Stack alignment requirement
3805 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3806
3807 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3808 // for calls to C. Supports the var-args backing area for register parms.
3809 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3810
3811 // The after-PROLOG location of the return address. Location of
3812 // return address specifies a type (REG or STACK) and a number
3813 // representing the register number (i.e. - use a register name) or
3814 // stack slot.
3815 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3816 // Otherwise, it is above the locks and verification slot and alignment word
3817 // TODO this may well be correct but need to check why that - 2 is there
3818 // ppc port uses 0 but we definitely need to allow for fixed_slots
3819 // which folds in the space used for monitors
3820 return_addr(STACK - 2 +
3821 align_up((Compile::current()->in_preserve_stack_slots() +
3822 Compile::current()->fixed_slots()),
3823 stack_alignment_in_slots()));
3824
3825 // Location of compiled Java return values. Same as C for now.
3826 return_value
3827 %{
3828 // TODO do we allow ideal_reg == Op_RegN???
3829 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3830 "only return normal values");
3831
3832 static const int lo[Op_RegL + 1] = { // enum name
3833 0, // Op_Node
3834 0, // Op_Set
3835 R0_num, // Op_RegN
3836 R0_num, // Op_RegI
3837 R0_num, // Op_RegP
3838 V0_num, // Op_RegF
3839 V0_num, // Op_RegD
3840 R0_num // Op_RegL
3841 };
3842
3843 static const int hi[Op_RegL + 1] = { // enum name
3844 0, // Op_Node
3845 0, // Op_Set
3846 OptoReg::Bad, // Op_RegN
3847 OptoReg::Bad, // Op_RegI
3848 R0_H_num, // Op_RegP
3849 OptoReg::Bad, // Op_RegF
3850 V0_H_num, // Op_RegD
3851 R0_H_num // Op_RegL
3852 };
3853
3854 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3855 %}
3856 %}
3857
3858 //----------ATTRIBUTES---------------------------------------------------------
3859 //----------Operand Attributes-------------------------------------------------
3860 op_attrib op_cost(1); // Required cost attribute
3861
3862 //----------Instruction Attributes---------------------------------------------
3863 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3864 ins_attrib ins_size(32); // Required size attribute (in bits)
3865 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3866 // a non-matching short branch variant
3867 // of some long branch?
3868 ins_attrib ins_alignment(4); // Required alignment attribute (must
3869 // be a power of 2) specifies the
3870 // alignment that some part of the
3871 // instruction (not necessarily the
3872 // start) requires. If > 1, a
3873 // compute_padding() function must be
3874 // provided for the instruction
3875
3876 // Whether this node is expanded during code emission into a sequence of
3877 // instructions and the first instruction can perform an implicit null check.
3878 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3879
3880 //----------OPERANDS-----------------------------------------------------------
3881 // Operand definitions must precede instruction definitions for correct parsing
3882 // in the ADLC because operands constitute user defined types which are used in
3883 // instruction definitions.
3884
3885 //----------Simple Operands----------------------------------------------------
3886
3887 // Integer operands 32 bit
3888 // 32 bit immediate
3889 operand immI()
3890 %{
3891 match(ConI);
3892
3893 op_cost(0);
3894 format %{ %}
3895 interface(CONST_INTER);
3896 %}
3897
3898 // 32 bit zero
3899 operand immI0()
3900 %{
3901 predicate(n->get_int() == 0);
3902 match(ConI);
3903
3904 op_cost(0);
3905 format %{ %}
3906 interface(CONST_INTER);
3907 %}
3908
3909 // 32 bit unit increment
3910 operand immI_1()
3911 %{
3912 predicate(n->get_int() == 1);
3913 match(ConI);
3914
3915 op_cost(0);
3916 format %{ %}
3917 interface(CONST_INTER);
3918 %}
3919
3920 // 32 bit unit decrement
3921 operand immI_M1()
3922 %{
3923 predicate(n->get_int() == -1);
3924 match(ConI);
3925
3926 op_cost(0);
3927 format %{ %}
3928 interface(CONST_INTER);
3929 %}
3930
3931 // Shift values for add/sub extension shift
3932 operand immIExt()
3933 %{
3934 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3935 match(ConI);
3936
3937 op_cost(0);
3938 format %{ %}
3939 interface(CONST_INTER);
3940 %}
3941
3942 operand immI_gt_1()
3943 %{
3944 predicate(n->get_int() > 1);
3945 match(ConI);
3946
3947 op_cost(0);
3948 format %{ %}
3949 interface(CONST_INTER);
3950 %}
3951
3952 operand immI_le_4()
3953 %{
3954 predicate(n->get_int() <= 4);
3955 match(ConI);
3956
3957 op_cost(0);
3958 format %{ %}
3959 interface(CONST_INTER);
3960 %}
3961
3962 operand immI_16()
3963 %{
3964 predicate(n->get_int() == 16);
3965 match(ConI);
3966
3967 op_cost(0);
3968 format %{ %}
3969 interface(CONST_INTER);
3970 %}
3971
3972 operand immI_24()
3973 %{
3974 predicate(n->get_int() == 24);
3975 match(ConI);
3976
3977 op_cost(0);
3978 format %{ %}
3979 interface(CONST_INTER);
3980 %}
3981
3982 operand immI_32()
3983 %{
3984 predicate(n->get_int() == 32);
3985 match(ConI);
3986
3987 op_cost(0);
3988 format %{ %}
3989 interface(CONST_INTER);
3990 %}
3991
3992 operand immI_48()
3993 %{
3994 predicate(n->get_int() == 48);
3995 match(ConI);
3996
3997 op_cost(0);
3998 format %{ %}
3999 interface(CONST_INTER);
4000 %}
4001
4002 operand immI_56()
4003 %{
4004 predicate(n->get_int() == 56);
4005 match(ConI);
4006
4007 op_cost(0);
4008 format %{ %}
4009 interface(CONST_INTER);
4010 %}
4011
4012 operand immI_255()
4013 %{
4014 predicate(n->get_int() == 255);
4015 match(ConI);
4016
4017 op_cost(0);
4018 format %{ %}
4019 interface(CONST_INTER);
4020 %}
4021
4022 operand immI_65535()
4023 %{
4024 predicate(n->get_int() == 65535);
4025 match(ConI);
4026
4027 op_cost(0);
4028 format %{ %}
4029 interface(CONST_INTER);
4030 %}
4031
4032 operand immI_positive()
4033 %{
4034 predicate(n->get_int() > 0);
4035 match(ConI);
4036
4037 op_cost(0);
4038 format %{ %}
4039 interface(CONST_INTER);
4040 %}
4041
4042 // BoolTest condition for signed compare
4043 operand immI_cmp_cond()
4044 %{
4045 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4046 match(ConI);
4047
4048 op_cost(0);
4049 format %{ %}
4050 interface(CONST_INTER);
4051 %}
4052
4053 // BoolTest condition for unsigned compare
4054 operand immI_cmpU_cond()
4055 %{
4056 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4057 match(ConI);
4058
4059 op_cost(0);
4060 format %{ %}
4061 interface(CONST_INTER);
4062 %}
4063
4064 operand immL_255()
4065 %{
4066 predicate(n->get_long() == 255L);
4067 match(ConL);
4068
4069 op_cost(0);
4070 format %{ %}
4071 interface(CONST_INTER);
4072 %}
4073
4074 operand immL_65535()
4075 %{
4076 predicate(n->get_long() == 65535L);
4077 match(ConL);
4078
4079 op_cost(0);
4080 format %{ %}
4081 interface(CONST_INTER);
4082 %}
4083
4084 operand immL_4294967295()
4085 %{
4086 predicate(n->get_long() == 4294967295L);
4087 match(ConL);
4088
4089 op_cost(0);
4090 format %{ %}
4091 interface(CONST_INTER);
4092 %}
4093
4094 operand immL_bitmask()
4095 %{
4096 predicate((n->get_long() != 0)
4097 && ((n->get_long() & 0xc000000000000000l) == 0)
4098 && is_power_of_2(n->get_long() + 1));
4099 match(ConL);
4100
4101 op_cost(0);
4102 format %{ %}
4103 interface(CONST_INTER);
4104 %}
4105
4106 operand immI_bitmask()
4107 %{
4108 predicate((n->get_int() != 0)
4109 && ((n->get_int() & 0xc0000000) == 0)
4110 && is_power_of_2(n->get_int() + 1));
4111 match(ConI);
4112
4113 op_cost(0);
4114 format %{ %}
4115 interface(CONST_INTER);
4116 %}
4117
4118 operand immL_positive_bitmaskI()
4119 %{
4120 predicate((n->get_long() != 0)
4121 && ((julong)n->get_long() < 0x80000000ULL)
4122 && is_power_of_2(n->get_long() + 1));
4123 match(ConL);
4124
4125 op_cost(0);
4126 format %{ %}
4127 interface(CONST_INTER);
4128 %}
4129
4130 // Scale values for scaled offset addressing modes (up to long but not quad)
4131 operand immIScale()
4132 %{
4133 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4134 match(ConI);
4135
4136 op_cost(0);
4137 format %{ %}
4138 interface(CONST_INTER);
4139 %}
4140
4141 // 5 bit signed integer
4142 operand immI5()
4143 %{
4144 predicate(Assembler::is_simm(n->get_int(), 5));
4145 match(ConI);
4146
4147 op_cost(0);
4148 format %{ %}
4149 interface(CONST_INTER);
4150 %}
4151
4152 // 7 bit unsigned integer
4153 operand immIU7()
4154 %{
4155 predicate(Assembler::is_uimm(n->get_int(), 7));
4156 match(ConI);
4157
4158 op_cost(0);
4159 format %{ %}
4160 interface(CONST_INTER);
4161 %}
4162
4163 // Offset for scaled or unscaled immediate loads and stores
4164 operand immIOffset()
4165 %{
4166 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4167 match(ConI);
4168
4169 op_cost(0);
4170 format %{ %}
4171 interface(CONST_INTER);
4172 %}
4173
4174 operand immIOffset1()
4175 %{
4176 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4177 match(ConI);
4178
4179 op_cost(0);
4180 format %{ %}
4181 interface(CONST_INTER);
4182 %}
4183
4184 operand immIOffset2()
4185 %{
4186 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4187 match(ConI);
4188
4189 op_cost(0);
4190 format %{ %}
4191 interface(CONST_INTER);
4192 %}
4193
4194 operand immIOffset4()
4195 %{
4196 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4197 match(ConI);
4198
4199 op_cost(0);
4200 format %{ %}
4201 interface(CONST_INTER);
4202 %}
4203
4204 operand immIOffset8()
4205 %{
4206 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4207 match(ConI);
4208
4209 op_cost(0);
4210 format %{ %}
4211 interface(CONST_INTER);
4212 %}
4213
4214 operand immIOffset16()
4215 %{
4216 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4217 match(ConI);
4218
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 operand immLOffset()
4225 %{
4226 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4227 match(ConL);
4228
4229 op_cost(0);
4230 format %{ %}
4231 interface(CONST_INTER);
4232 %}
4233
4234 operand immLoffset1()
4235 %{
4236 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4237 match(ConL);
4238
4239 op_cost(0);
4240 format %{ %}
4241 interface(CONST_INTER);
4242 %}
4243
4244 operand immLoffset2()
4245 %{
4246 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4247 match(ConL);
4248
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 operand immLoffset4()
4255 %{
4256 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4257 match(ConL);
4258
4259 op_cost(0);
4260 format %{ %}
4261 interface(CONST_INTER);
4262 %}
4263
4264 operand immLoffset8()
4265 %{
4266 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4267 match(ConL);
4268
4269 op_cost(0);
4270 format %{ %}
4271 interface(CONST_INTER);
4272 %}
4273
4274 operand immLoffset16()
4275 %{
4276 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4277 match(ConL);
4278
4279 op_cost(0);
4280 format %{ %}
4281 interface(CONST_INTER);
4282 %}
4283
4284 // 5 bit signed long integer
4285 operand immL5()
4286 %{
4287 predicate(Assembler::is_simm(n->get_long(), 5));
4288 match(ConL);
4289
4290 op_cost(0);
4291 format %{ %}
4292 interface(CONST_INTER);
4293 %}
4294
4295 // 7 bit unsigned long integer
4296 operand immLU7()
4297 %{
4298 predicate(Assembler::is_uimm(n->get_long(), 7));
4299 match(ConL);
4300
4301 op_cost(0);
4302 format %{ %}
4303 interface(CONST_INTER);
4304 %}
4305
4306 // 8 bit signed value.
4307 operand immI8()
4308 %{
4309 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4310 match(ConI);
4311
4312 op_cost(0);
4313 format %{ %}
4314 interface(CONST_INTER);
4315 %}
4316
4317 // 8 bit signed value (simm8), or #simm8 LSL 8.
4318 operand immIDupV()
4319 %{
4320 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4321 match(ConI);
4322
4323 op_cost(0);
4324 format %{ %}
4325 interface(CONST_INTER);
4326 %}
4327
4328 // 8 bit signed value (simm8), or #simm8 LSL 8.
4329 operand immLDupV()
4330 %{
4331 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4332 match(ConL);
4333
4334 op_cost(0);
4335 format %{ %}
4336 interface(CONST_INTER);
4337 %}
4338
4339 // 8 bit signed value (simm8), or #simm8 LSL 8.
4340 operand immHDupV()
4341 %{
4342 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4343 match(ConH);
4344
4345 op_cost(0);
4346 format %{ %}
4347 interface(CONST_INTER);
4348 %}
4349
4350 // 8 bit integer valid for vector add sub immediate
4351 operand immBAddSubV()
4352 %{
4353 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4354 match(ConI);
4355
4356 op_cost(0);
4357 format %{ %}
4358 interface(CONST_INTER);
4359 %}
4360
4361 // 32 bit integer valid for add sub immediate
4362 operand immIAddSub()
4363 %{
4364 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4365 match(ConI);
4366 op_cost(0);
4367 format %{ %}
4368 interface(CONST_INTER);
4369 %}
4370
4371 // 32 bit integer valid for vector add sub immediate
4372 operand immIAddSubV()
4373 %{
4374 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4375 match(ConI);
4376
4377 op_cost(0);
4378 format %{ %}
4379 interface(CONST_INTER);
4380 %}
4381
4382 // 32 bit unsigned integer valid for logical immediate
4383
4384 operand immBLog()
4385 %{
4386 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4387 match(ConI);
4388
4389 op_cost(0);
4390 format %{ %}
4391 interface(CONST_INTER);
4392 %}
4393
4394 operand immSLog()
4395 %{
4396 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4397 match(ConI);
4398
4399 op_cost(0);
4400 format %{ %}
4401 interface(CONST_INTER);
4402 %}
4403
4404 operand immILog()
4405 %{
4406 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4407 match(ConI);
4408
4409 op_cost(0);
4410 format %{ %}
4411 interface(CONST_INTER);
4412 %}
4413
4414 // Integer operands 64 bit
4415 // 64 bit immediate
4416 operand immL()
4417 %{
4418 match(ConL);
4419
4420 op_cost(0);
4421 format %{ %}
4422 interface(CONST_INTER);
4423 %}
4424
4425 // 64 bit zero
4426 operand immL0()
4427 %{
4428 predicate(n->get_long() == 0);
4429 match(ConL);
4430
4431 op_cost(0);
4432 format %{ %}
4433 interface(CONST_INTER);
4434 %}
4435
4436 // 64 bit unit decrement
4437 operand immL_M1()
4438 %{
4439 predicate(n->get_long() == -1);
4440 match(ConL);
4441
4442 op_cost(0);
4443 format %{ %}
4444 interface(CONST_INTER);
4445 %}
4446
4447 // 64 bit integer valid for add sub immediate
4448 operand immLAddSub()
4449 %{
4450 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4451 match(ConL);
4452 op_cost(0);
4453 format %{ %}
4454 interface(CONST_INTER);
4455 %}
4456
4457 // 64 bit integer valid for addv subv immediate
4458 operand immLAddSubV()
4459 %{
4460 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4461 match(ConL);
4462
4463 op_cost(0);
4464 format %{ %}
4465 interface(CONST_INTER);
4466 %}
4467
4468 // 64 bit integer valid for logical immediate
4469 operand immLLog()
4470 %{
4471 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4472 match(ConL);
4473 op_cost(0);
4474 format %{ %}
4475 interface(CONST_INTER);
4476 %}
4477
4478 // Long Immediate: low 32-bit mask
4479 operand immL_32bits()
4480 %{
4481 predicate(n->get_long() == 0xFFFFFFFFL);
4482 match(ConL);
4483 op_cost(0);
4484 format %{ %}
4485 interface(CONST_INTER);
4486 %}
4487
4488 // Pointer operands
4489 // Pointer Immediate
4490 operand immP()
4491 %{
4492 match(ConP);
4493
4494 op_cost(0);
4495 format %{ %}
4496 interface(CONST_INTER);
4497 %}
4498
4499 // nullptr Pointer Immediate
4500 operand immP0()
4501 %{
4502 predicate(n->get_ptr() == 0);
4503 match(ConP);
4504
4505 op_cost(0);
4506 format %{ %}
4507 interface(CONST_INTER);
4508 %}
4509
4510 // Pointer Immediate One
4511 // this is used in object initialization (initial object header)
4512 operand immP_1()
4513 %{
4514 predicate(n->get_ptr() == 1);
4515 match(ConP);
4516
4517 op_cost(0);
4518 format %{ %}
4519 interface(CONST_INTER);
4520 %}
4521
4522 // AOT Runtime Constants Address
4523 operand immAOTRuntimeConstantsAddress()
4524 %{
4525 // Check if the address is in the range of AOT Runtime Constants
4526 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4527 match(ConP);
4528
4529 op_cost(0);
4530 format %{ %}
4531 interface(CONST_INTER);
4532 %}
4533
4534 // Float and Double operands
4535 // Double Immediate
4536 operand immD()
4537 %{
4538 match(ConD);
4539 op_cost(0);
4540 format %{ %}
4541 interface(CONST_INTER);
4542 %}
4543
4544 // Double Immediate: +0.0d
4545 operand immD0()
4546 %{
4547 predicate(jlong_cast(n->getd()) == 0);
4548 match(ConD);
4549
4550 op_cost(0);
4551 format %{ %}
4552 interface(CONST_INTER);
4553 %}
4554
4555 // constant 'double +0.0'.
4556 operand immDPacked()
4557 %{
4558 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4559 match(ConD);
4560 op_cost(0);
4561 format %{ %}
4562 interface(CONST_INTER);
4563 %}
4564
4565 // Float Immediate
4566 operand immF()
4567 %{
4568 match(ConF);
4569 op_cost(0);
4570 format %{ %}
4571 interface(CONST_INTER);
4572 %}
4573
4574 // Float Immediate: +0.0f.
4575 operand immF0()
4576 %{
4577 predicate(jint_cast(n->getf()) == 0);
4578 match(ConF);
4579
4580 op_cost(0);
4581 format %{ %}
4582 interface(CONST_INTER);
4583 %}
4584
4585 // Half Float (FP16) Immediate
4586 operand immH()
4587 %{
4588 match(ConH);
4589 op_cost(0);
4590 format %{ %}
4591 interface(CONST_INTER);
4592 %}
4593
4594 //
4595 operand immFPacked()
4596 %{
4597 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4598 match(ConF);
4599 op_cost(0);
4600 format %{ %}
4601 interface(CONST_INTER);
4602 %}
4603
4604 // Narrow pointer operands
4605 // Narrow Pointer Immediate
4606 operand immN()
4607 %{
4608 match(ConN);
4609
4610 op_cost(0);
4611 format %{ %}
4612 interface(CONST_INTER);
4613 %}
4614
4615 // Narrow nullptr Pointer Immediate
4616 operand immN0()
4617 %{
4618 predicate(n->get_narrowcon() == 0);
4619 match(ConN);
4620
4621 op_cost(0);
4622 format %{ %}
4623 interface(CONST_INTER);
4624 %}
4625
4626 operand immNKlass()
4627 %{
4628 match(ConNKlass);
4629
4630 op_cost(0);
4631 format %{ %}
4632 interface(CONST_INTER);
4633 %}
4634
4635 // Integer 32 bit Register Operands
4636 // Integer 32 bitRegister (excludes SP)
4637 operand iRegI()
4638 %{
4639 constraint(ALLOC_IN_RC(any_reg32));
4640 match(RegI);
4641 match(iRegINoSp);
4642 op_cost(0);
4643 format %{ %}
4644 interface(REG_INTER);
4645 %}
4646
4647 // Integer 32 bit Register not Special
4648 operand iRegINoSp()
4649 %{
4650 constraint(ALLOC_IN_RC(no_special_reg32));
4651 match(RegI);
4652 op_cost(0);
4653 format %{ %}
4654 interface(REG_INTER);
4655 %}
4656
4657 // Integer 64 bit Register Operands
4658 // Integer 64 bit Register (includes SP)
4659 operand iRegL()
4660 %{
4661 constraint(ALLOC_IN_RC(any_reg));
4662 match(RegL);
4663 match(iRegLNoSp);
4664 op_cost(0);
4665 format %{ %}
4666 interface(REG_INTER);
4667 %}
4668
4669 // Integer 64 bit Register not Special
4670 operand iRegLNoSp()
4671 %{
4672 constraint(ALLOC_IN_RC(no_special_reg));
4673 match(RegL);
4674 match(iRegL_R0);
4675 format %{ %}
4676 interface(REG_INTER);
4677 %}
4678
4679 // Pointer Register Operands
4680 // Pointer Register
4681 operand iRegP()
4682 %{
4683 constraint(ALLOC_IN_RC(ptr_reg));
4684 match(RegP);
4685 match(iRegPNoSp);
4686 match(iRegP_R0);
4687 //match(iRegP_R2);
4688 //match(iRegP_R4);
4689 match(iRegP_R5);
4690 match(thread_RegP);
4691 op_cost(0);
4692 format %{ %}
4693 interface(REG_INTER);
4694 %}
4695
4696 // Pointer 64 bit Register not Special
4697 operand iRegPNoSp()
4698 %{
4699 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4700 match(RegP);
4701 // match(iRegP);
4702 // match(iRegP_R0);
4703 // match(iRegP_R2);
4704 // match(iRegP_R4);
4705 // match(iRegP_R5);
4706 // match(thread_RegP);
4707 op_cost(0);
4708 format %{ %}
4709 interface(REG_INTER);
4710 %}
4711
4712 // This operand is not allowed to use rfp even if
4713 // rfp is not used to hold the frame pointer.
4714 operand iRegPNoSpNoRfp()
4715 %{
4716 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4717 match(RegP);
4718 match(iRegPNoSp);
4719 op_cost(0);
4720 format %{ %}
4721 interface(REG_INTER);
4722 %}
4723
4724 // Pointer 64 bit Register R0 only
4725 operand iRegP_R0()
4726 %{
4727 constraint(ALLOC_IN_RC(r0_reg));
4728 match(RegP);
4729 // match(iRegP);
4730 match(iRegPNoSp);
4731 op_cost(0);
4732 format %{ %}
4733 interface(REG_INTER);
4734 %}
4735
4736 // Pointer 64 bit Register R1 only
4737 operand iRegP_R1()
4738 %{
4739 constraint(ALLOC_IN_RC(r1_reg));
4740 match(RegP);
4741 // match(iRegP);
4742 match(iRegPNoSp);
4743 op_cost(0);
4744 format %{ %}
4745 interface(REG_INTER);
4746 %}
4747
4748 // Pointer 64 bit Register R2 only
4749 operand iRegP_R2()
4750 %{
4751 constraint(ALLOC_IN_RC(r2_reg));
4752 match(RegP);
4753 // match(iRegP);
4754 match(iRegPNoSp);
4755 op_cost(0);
4756 format %{ %}
4757 interface(REG_INTER);
4758 %}
4759
4760 // Pointer 64 bit Register R3 only
4761 operand iRegP_R3()
4762 %{
4763 constraint(ALLOC_IN_RC(r3_reg));
4764 match(RegP);
4765 // match(iRegP);
4766 match(iRegPNoSp);
4767 op_cost(0);
4768 format %{ %}
4769 interface(REG_INTER);
4770 %}
4771
4772 // Pointer 64 bit Register R4 only
4773 operand iRegP_R4()
4774 %{
4775 constraint(ALLOC_IN_RC(r4_reg));
4776 match(RegP);
4777 // match(iRegP);
4778 match(iRegPNoSp);
4779 op_cost(0);
4780 format %{ %}
4781 interface(REG_INTER);
4782 %}
4783
4784 // Pointer 64 bit Register R5 only
4785 operand iRegP_R5()
4786 %{
4787 constraint(ALLOC_IN_RC(r5_reg));
4788 match(RegP);
4789 // match(iRegP);
4790 match(iRegPNoSp);
4791 op_cost(0);
4792 format %{ %}
4793 interface(REG_INTER);
4794 %}
4795
4796 // Pointer 64 bit Register R10 only
4797 operand iRegP_R10()
4798 %{
4799 constraint(ALLOC_IN_RC(r10_reg));
4800 match(RegP);
4801 // match(iRegP);
4802 match(iRegPNoSp);
4803 op_cost(0);
4804 format %{ %}
4805 interface(REG_INTER);
4806 %}
4807
4808 // Long 64 bit Register R0 only
4809 operand iRegL_R0()
4810 %{
4811 constraint(ALLOC_IN_RC(r0_reg));
4812 match(RegL);
4813 match(iRegLNoSp);
4814 op_cost(0);
4815 format %{ %}
4816 interface(REG_INTER);
4817 %}
4818
4819 // Long 64 bit Register R11 only
4820 operand iRegL_R11()
4821 %{
4822 constraint(ALLOC_IN_RC(r11_reg));
4823 match(RegL);
4824 match(iRegLNoSp);
4825 op_cost(0);
4826 format %{ %}
4827 interface(REG_INTER);
4828 %}
4829
4830 // Register R0 only
4831 operand iRegI_R0()
4832 %{
4833 constraint(ALLOC_IN_RC(int_r0_reg));
4834 match(RegI);
4835 match(iRegINoSp);
4836 op_cost(0);
4837 format %{ %}
4838 interface(REG_INTER);
4839 %}
4840
4841 // Register R2 only
4842 operand iRegI_R2()
4843 %{
4844 constraint(ALLOC_IN_RC(int_r2_reg));
4845 match(RegI);
4846 match(iRegINoSp);
4847 op_cost(0);
4848 format %{ %}
4849 interface(REG_INTER);
4850 %}
4851
4852 // Register R3 only
4853 operand iRegI_R3()
4854 %{
4855 constraint(ALLOC_IN_RC(int_r3_reg));
4856 match(RegI);
4857 match(iRegINoSp);
4858 op_cost(0);
4859 format %{ %}
4860 interface(REG_INTER);
4861 %}
4862
4863
4864 // Register R4 only
4865 operand iRegI_R4()
4866 %{
4867 constraint(ALLOC_IN_RC(int_r4_reg));
4868 match(RegI);
4869 match(iRegINoSp);
4870 op_cost(0);
4871 format %{ %}
4872 interface(REG_INTER);
4873 %}
4874
4875
4876 // Pointer Register Operands
4877 // Narrow Pointer Register
4878 operand iRegN()
4879 %{
4880 constraint(ALLOC_IN_RC(any_reg32));
4881 match(RegN);
4882 match(iRegNNoSp);
4883 op_cost(0);
4884 format %{ %}
4885 interface(REG_INTER);
4886 %}
4887
4888 // Integer 64 bit Register not Special
4889 operand iRegNNoSp()
4890 %{
4891 constraint(ALLOC_IN_RC(no_special_reg32));
4892 match(RegN);
4893 op_cost(0);
4894 format %{ %}
4895 interface(REG_INTER);
4896 %}
4897
4898 // Float Register
4899 // Float register operands
4900 operand vRegF()
4901 %{
4902 constraint(ALLOC_IN_RC(float_reg));
4903 match(RegF);
4904
4905 op_cost(0);
4906 format %{ %}
4907 interface(REG_INTER);
4908 %}
4909
4910 // Double Register
4911 // Double register operands
4912 operand vRegD()
4913 %{
4914 constraint(ALLOC_IN_RC(double_reg));
4915 match(RegD);
4916
4917 op_cost(0);
4918 format %{ %}
4919 interface(REG_INTER);
4920 %}
4921
4922 // Generic vector class. This will be used for
4923 // all vector operands, including NEON and SVE.
4924 operand vReg()
4925 %{
4926 constraint(ALLOC_IN_RC(dynamic));
4927 match(VecA);
4928 match(VecD);
4929 match(VecX);
4930
4931 op_cost(0);
4932 format %{ %}
4933 interface(REG_INTER);
4934 %}
4935
4936 operand vReg_V10()
4937 %{
4938 constraint(ALLOC_IN_RC(v10_veca_reg));
4939 match(vReg);
4940
4941 op_cost(0);
4942 format %{ %}
4943 interface(REG_INTER);
4944 %}
4945
4946 operand vReg_V11()
4947 %{
4948 constraint(ALLOC_IN_RC(v11_veca_reg));
4949 match(vReg);
4950
4951 op_cost(0);
4952 format %{ %}
4953 interface(REG_INTER);
4954 %}
4955
4956 operand vReg_V12()
4957 %{
4958 constraint(ALLOC_IN_RC(v12_veca_reg));
4959 match(vReg);
4960
4961 op_cost(0);
4962 format %{ %}
4963 interface(REG_INTER);
4964 %}
4965
4966 operand vReg_V13()
4967 %{
4968 constraint(ALLOC_IN_RC(v13_veca_reg));
4969 match(vReg);
4970
4971 op_cost(0);
4972 format %{ %}
4973 interface(REG_INTER);
4974 %}
4975
4976 operand vReg_V17()
4977 %{
4978 constraint(ALLOC_IN_RC(v17_veca_reg));
4979 match(vReg);
4980
4981 op_cost(0);
4982 format %{ %}
4983 interface(REG_INTER);
4984 %}
4985
4986 operand vReg_V18()
4987 %{
4988 constraint(ALLOC_IN_RC(v18_veca_reg));
4989 match(vReg);
4990
4991 op_cost(0);
4992 format %{ %}
4993 interface(REG_INTER);
4994 %}
4995
4996 operand vReg_V23()
4997 %{
4998 constraint(ALLOC_IN_RC(v23_veca_reg));
4999 match(vReg);
5000
5001 op_cost(0);
5002 format %{ %}
5003 interface(REG_INTER);
5004 %}
5005
5006 operand vReg_V24()
5007 %{
5008 constraint(ALLOC_IN_RC(v24_veca_reg));
5009 match(vReg);
5010
5011 op_cost(0);
5012 format %{ %}
5013 interface(REG_INTER);
5014 %}
5015
5016 operand vecA()
5017 %{
5018 constraint(ALLOC_IN_RC(vectora_reg));
5019 match(VecA);
5020
5021 op_cost(0);
5022 format %{ %}
5023 interface(REG_INTER);
5024 %}
5025
5026 operand vecD()
5027 %{
5028 constraint(ALLOC_IN_RC(vectord_reg));
5029 match(VecD);
5030
5031 op_cost(0);
5032 format %{ %}
5033 interface(REG_INTER);
5034 %}
5035
5036 operand vecX()
5037 %{
5038 constraint(ALLOC_IN_RC(vectorx_reg));
5039 match(VecX);
5040
5041 op_cost(0);
5042 format %{ %}
5043 interface(REG_INTER);
5044 %}
5045
5046 operand vRegD_V0()
5047 %{
5048 constraint(ALLOC_IN_RC(v0_reg));
5049 match(RegD);
5050 op_cost(0);
5051 format %{ %}
5052 interface(REG_INTER);
5053 %}
5054
5055 operand vRegD_V1()
5056 %{
5057 constraint(ALLOC_IN_RC(v1_reg));
5058 match(RegD);
5059 op_cost(0);
5060 format %{ %}
5061 interface(REG_INTER);
5062 %}
5063
5064 operand vRegD_V2()
5065 %{
5066 constraint(ALLOC_IN_RC(v2_reg));
5067 match(RegD);
5068 op_cost(0);
5069 format %{ %}
5070 interface(REG_INTER);
5071 %}
5072
5073 operand vRegD_V3()
5074 %{
5075 constraint(ALLOC_IN_RC(v3_reg));
5076 match(RegD);
5077 op_cost(0);
5078 format %{ %}
5079 interface(REG_INTER);
5080 %}
5081
5082 operand vRegD_V4()
5083 %{
5084 constraint(ALLOC_IN_RC(v4_reg));
5085 match(RegD);
5086 op_cost(0);
5087 format %{ %}
5088 interface(REG_INTER);
5089 %}
5090
5091 operand vRegD_V5()
5092 %{
5093 constraint(ALLOC_IN_RC(v5_reg));
5094 match(RegD);
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand vRegD_V6()
5101 %{
5102 constraint(ALLOC_IN_RC(v6_reg));
5103 match(RegD);
5104 op_cost(0);
5105 format %{ %}
5106 interface(REG_INTER);
5107 %}
5108
5109 operand vRegD_V7()
5110 %{
5111 constraint(ALLOC_IN_RC(v7_reg));
5112 match(RegD);
5113 op_cost(0);
5114 format %{ %}
5115 interface(REG_INTER);
5116 %}
5117
5118 operand vRegD_V12()
5119 %{
5120 constraint(ALLOC_IN_RC(v12_reg));
5121 match(RegD);
5122 op_cost(0);
5123 format %{ %}
5124 interface(REG_INTER);
5125 %}
5126
5127 operand vRegD_V13()
5128 %{
5129 constraint(ALLOC_IN_RC(v13_reg));
5130 match(RegD);
5131 op_cost(0);
5132 format %{ %}
5133 interface(REG_INTER);
5134 %}
5135
5136 operand pReg()
5137 %{
5138 constraint(ALLOC_IN_RC(pr_reg));
5139 match(RegVectMask);
5140 match(pRegGov);
5141 op_cost(0);
5142 format %{ %}
5143 interface(REG_INTER);
5144 %}
5145
5146 operand pRegGov()
5147 %{
5148 constraint(ALLOC_IN_RC(gov_pr));
5149 match(RegVectMask);
5150 match(pReg);
5151 op_cost(0);
5152 format %{ %}
5153 interface(REG_INTER);
5154 %}
5155
5156 operand pRegGov_P0()
5157 %{
5158 constraint(ALLOC_IN_RC(p0_reg));
5159 match(RegVectMask);
5160 op_cost(0);
5161 format %{ %}
5162 interface(REG_INTER);
5163 %}
5164
5165 operand pRegGov_P1()
5166 %{
5167 constraint(ALLOC_IN_RC(p1_reg));
5168 match(RegVectMask);
5169 op_cost(0);
5170 format %{ %}
5171 interface(REG_INTER);
5172 %}
5173
5174 // Flags register, used as output of signed compare instructions
5175
5176 // note that on AArch64 we also use this register as the output for
5177 // for floating point compare instructions (CmpF CmpD). this ensures
5178 // that ordered inequality tests use GT, GE, LT or LE none of which
5179 // pass through cases where the result is unordered i.e. one or both
5180 // inputs to the compare is a NaN. this means that the ideal code can
5181 // replace e.g. a GT with an LE and not end up capturing the NaN case
5182 // (where the comparison should always fail). EQ and NE tests are
5183 // always generated in ideal code so that unordered folds into the NE
5184 // case, matching the behaviour of AArch64 NE.
5185 //
5186 // This differs from x86 where the outputs of FP compares use a
5187 // special FP flags registers and where compares based on this
5188 // register are distinguished into ordered inequalities (cmpOpUCF) and
5189 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5190 // to explicitly handle the unordered case in branches. x86 also has
5191 // to include extra CMoveX rules to accept a cmpOpUCF input.
5192
5193 operand rFlagsReg()
5194 %{
5195 constraint(ALLOC_IN_RC(int_flags));
5196 match(RegFlags);
5197
5198 op_cost(0);
5199 format %{ "RFLAGS" %}
5200 interface(REG_INTER);
5201 %}
5202
5203 // Flags register, used as output of unsigned compare instructions
5204 operand rFlagsRegU()
5205 %{
5206 constraint(ALLOC_IN_RC(int_flags));
5207 match(RegFlags);
5208
5209 op_cost(0);
5210 format %{ "RFLAGSU" %}
5211 interface(REG_INTER);
5212 %}
5213
5214 // Special Registers
5215
5216 // Method Register
5217 operand inline_cache_RegP(iRegP reg)
5218 %{
5219 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5220 match(reg);
5221 match(iRegPNoSp);
5222 op_cost(0);
5223 format %{ %}
5224 interface(REG_INTER);
5225 %}
5226
5227 // Thread Register
5228 operand thread_RegP(iRegP reg)
5229 %{
5230 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5231 match(reg);
5232 op_cost(0);
5233 format %{ %}
5234 interface(REG_INTER);
5235 %}
5236
5237 //----------Memory Operands----------------------------------------------------
5238
5239 operand indirect(iRegP reg)
5240 %{
5241 constraint(ALLOC_IN_RC(ptr_reg));
5242 match(reg);
5243 op_cost(0);
5244 format %{ "[$reg]" %}
5245 interface(MEMORY_INTER) %{
5246 base($reg);
5247 index(0xffffffff);
5248 scale(0x0);
5249 disp(0x0);
5250 %}
5251 %}
5252
5253 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5254 %{
5255 constraint(ALLOC_IN_RC(ptr_reg));
5256 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5257 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5258 op_cost(0);
5259 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5260 interface(MEMORY_INTER) %{
5261 base($reg);
5262 index($ireg);
5263 scale($scale);
5264 disp(0x0);
5265 %}
5266 %}
5267
5268 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5269 %{
5270 constraint(ALLOC_IN_RC(ptr_reg));
5271 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5272 match(AddP reg (LShiftL lreg scale));
5273 op_cost(0);
5274 format %{ "$reg, $lreg lsl($scale)" %}
5275 interface(MEMORY_INTER) %{
5276 base($reg);
5277 index($lreg);
5278 scale($scale);
5279 disp(0x0);
5280 %}
5281 %}
5282
5283 operand indIndexI2L(iRegP reg, iRegI ireg)
5284 %{
5285 constraint(ALLOC_IN_RC(ptr_reg));
5286 match(AddP reg (ConvI2L ireg));
5287 op_cost(0);
5288 format %{ "$reg, $ireg, 0, I2L" %}
5289 interface(MEMORY_INTER) %{
5290 base($reg);
5291 index($ireg);
5292 scale(0x0);
5293 disp(0x0);
5294 %}
5295 %}
5296
5297 operand indIndex(iRegP reg, iRegL lreg)
5298 %{
5299 constraint(ALLOC_IN_RC(ptr_reg));
5300 match(AddP reg lreg);
5301 op_cost(0);
5302 format %{ "$reg, $lreg" %}
5303 interface(MEMORY_INTER) %{
5304 base($reg);
5305 index($lreg);
5306 scale(0x0);
5307 disp(0x0);
5308 %}
5309 %}
5310
5311 operand indOffI1(iRegP reg, immIOffset1 off)
5312 %{
5313 constraint(ALLOC_IN_RC(ptr_reg));
5314 match(AddP reg off);
5315 op_cost(0);
5316 format %{ "[$reg, $off]" %}
5317 interface(MEMORY_INTER) %{
5318 base($reg);
5319 index(0xffffffff);
5320 scale(0x0);
5321 disp($off);
5322 %}
5323 %}
5324
5325 operand indOffI2(iRegP reg, immIOffset2 off)
5326 %{
5327 constraint(ALLOC_IN_RC(ptr_reg));
5328 match(AddP reg off);
5329 op_cost(0);
5330 format %{ "[$reg, $off]" %}
5331 interface(MEMORY_INTER) %{
5332 base($reg);
5333 index(0xffffffff);
5334 scale(0x0);
5335 disp($off);
5336 %}
5337 %}
5338
5339 operand indOffI4(iRegP reg, immIOffset4 off)
5340 %{
5341 constraint(ALLOC_IN_RC(ptr_reg));
5342 match(AddP reg off);
5343 op_cost(0);
5344 format %{ "[$reg, $off]" %}
5345 interface(MEMORY_INTER) %{
5346 base($reg);
5347 index(0xffffffff);
5348 scale(0x0);
5349 disp($off);
5350 %}
5351 %}
5352
5353 operand indOffI8(iRegP reg, immIOffset8 off)
5354 %{
5355 constraint(ALLOC_IN_RC(ptr_reg));
5356 match(AddP reg off);
5357 op_cost(0);
5358 format %{ "[$reg, $off]" %}
5359 interface(MEMORY_INTER) %{
5360 base($reg);
5361 index(0xffffffff);
5362 scale(0x0);
5363 disp($off);
5364 %}
5365 %}
5366
5367 operand indOffI16(iRegP reg, immIOffset16 off)
5368 %{
5369 constraint(ALLOC_IN_RC(ptr_reg));
5370 match(AddP reg off);
5371 op_cost(0);
5372 format %{ "[$reg, $off]" %}
5373 interface(MEMORY_INTER) %{
5374 base($reg);
5375 index(0xffffffff);
5376 scale(0x0);
5377 disp($off);
5378 %}
5379 %}
5380
5381 operand indOffL1(iRegP reg, immLoffset1 off)
5382 %{
5383 constraint(ALLOC_IN_RC(ptr_reg));
5384 match(AddP reg off);
5385 op_cost(0);
5386 format %{ "[$reg, $off]" %}
5387 interface(MEMORY_INTER) %{
5388 base($reg);
5389 index(0xffffffff);
5390 scale(0x0);
5391 disp($off);
5392 %}
5393 %}
5394
5395 operand indOffL2(iRegP reg, immLoffset2 off)
5396 %{
5397 constraint(ALLOC_IN_RC(ptr_reg));
5398 match(AddP reg off);
5399 op_cost(0);
5400 format %{ "[$reg, $off]" %}
5401 interface(MEMORY_INTER) %{
5402 base($reg);
5403 index(0xffffffff);
5404 scale(0x0);
5405 disp($off);
5406 %}
5407 %}
5408
5409 operand indOffL4(iRegP reg, immLoffset4 off)
5410 %{
5411 constraint(ALLOC_IN_RC(ptr_reg));
5412 match(AddP reg off);
5413 op_cost(0);
5414 format %{ "[$reg, $off]" %}
5415 interface(MEMORY_INTER) %{
5416 base($reg);
5417 index(0xffffffff);
5418 scale(0x0);
5419 disp($off);
5420 %}
5421 %}
5422
5423 operand indOffL8(iRegP reg, immLoffset8 off)
5424 %{
5425 constraint(ALLOC_IN_RC(ptr_reg));
5426 match(AddP reg off);
5427 op_cost(0);
5428 format %{ "[$reg, $off]" %}
5429 interface(MEMORY_INTER) %{
5430 base($reg);
5431 index(0xffffffff);
5432 scale(0x0);
5433 disp($off);
5434 %}
5435 %}
5436
5437 operand indOffL16(iRegP reg, immLoffset16 off)
5438 %{
5439 constraint(ALLOC_IN_RC(ptr_reg));
5440 match(AddP reg off);
5441 op_cost(0);
5442 format %{ "[$reg, $off]" %}
5443 interface(MEMORY_INTER) %{
5444 base($reg);
5445 index(0xffffffff);
5446 scale(0x0);
5447 disp($off);
5448 %}
5449 %}
5450
5451 operand indirectX2P(iRegL reg)
5452 %{
5453 constraint(ALLOC_IN_RC(ptr_reg));
5454 match(CastX2P reg);
5455 op_cost(0);
5456 format %{ "[$reg]\t# long -> ptr" %}
5457 interface(MEMORY_INTER) %{
5458 base($reg);
5459 index(0xffffffff);
5460 scale(0x0);
5461 disp(0x0);
5462 %}
5463 %}
5464
5465 operand indOffX2P(iRegL reg, immLOffset off)
5466 %{
5467 constraint(ALLOC_IN_RC(ptr_reg));
5468 match(AddP (CastX2P reg) off);
5469 op_cost(0);
5470 format %{ "[$reg, $off]\t# long -> ptr" %}
5471 interface(MEMORY_INTER) %{
5472 base($reg);
5473 index(0xffffffff);
5474 scale(0x0);
5475 disp($off);
5476 %}
5477 %}
5478
5479 operand indirectN(iRegN reg)
5480 %{
5481 predicate(CompressedOops::shift() == 0);
5482 constraint(ALLOC_IN_RC(ptr_reg));
5483 match(DecodeN reg);
5484 op_cost(0);
5485 format %{ "[$reg]\t# narrow" %}
5486 interface(MEMORY_INTER) %{
5487 base($reg);
5488 index(0xffffffff);
5489 scale(0x0);
5490 disp(0x0);
5491 %}
5492 %}
5493
5494 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5495 %{
5496 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5497 constraint(ALLOC_IN_RC(ptr_reg));
5498 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5499 op_cost(0);
5500 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5501 interface(MEMORY_INTER) %{
5502 base($reg);
5503 index($ireg);
5504 scale($scale);
5505 disp(0x0);
5506 %}
5507 %}
5508
5509 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5510 %{
5511 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5512 constraint(ALLOC_IN_RC(ptr_reg));
5513 match(AddP (DecodeN reg) (LShiftL lreg scale));
5514 op_cost(0);
5515 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5516 interface(MEMORY_INTER) %{
5517 base($reg);
5518 index($lreg);
5519 scale($scale);
5520 disp(0x0);
5521 %}
5522 %}
5523
5524 operand indIndexI2LN(iRegN reg, iRegI ireg)
5525 %{
5526 predicate(CompressedOops::shift() == 0);
5527 constraint(ALLOC_IN_RC(ptr_reg));
5528 match(AddP (DecodeN reg) (ConvI2L ireg));
5529 op_cost(0);
5530 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5531 interface(MEMORY_INTER) %{
5532 base($reg);
5533 index($ireg);
5534 scale(0x0);
5535 disp(0x0);
5536 %}
5537 %}
5538
5539 operand indIndexN(iRegN reg, iRegL lreg)
5540 %{
5541 predicate(CompressedOops::shift() == 0);
5542 constraint(ALLOC_IN_RC(ptr_reg));
5543 match(AddP (DecodeN reg) lreg);
5544 op_cost(0);
5545 format %{ "$reg, $lreg\t# narrow" %}
5546 interface(MEMORY_INTER) %{
5547 base($reg);
5548 index($lreg);
5549 scale(0x0);
5550 disp(0x0);
5551 %}
5552 %}
5553
5554 operand indOffIN(iRegN reg, immIOffset off)
5555 %{
5556 predicate(CompressedOops::shift() == 0);
5557 constraint(ALLOC_IN_RC(ptr_reg));
5558 match(AddP (DecodeN reg) off);
5559 op_cost(0);
5560 format %{ "[$reg, $off]\t# narrow" %}
5561 interface(MEMORY_INTER) %{
5562 base($reg);
5563 index(0xffffffff);
5564 scale(0x0);
5565 disp($off);
5566 %}
5567 %}
5568
5569 operand indOffLN(iRegN reg, immLOffset off)
5570 %{
5571 predicate(CompressedOops::shift() == 0);
5572 constraint(ALLOC_IN_RC(ptr_reg));
5573 match(AddP (DecodeN reg) off);
5574 op_cost(0);
5575 format %{ "[$reg, $off]\t# narrow" %}
5576 interface(MEMORY_INTER) %{
5577 base($reg);
5578 index(0xffffffff);
5579 scale(0x0);
5580 disp($off);
5581 %}
5582 %}
5583
5584
5585 //----------Special Memory Operands--------------------------------------------
5586 // Stack Slot Operand - This operand is used for loading and storing temporary
5587 // values on the stack where a match requires a value to
5588 // flow through memory.
5589 operand stackSlotP(sRegP reg)
5590 %{
5591 constraint(ALLOC_IN_RC(stack_slots));
5592 op_cost(100);
5593 // No match rule because this operand is only generated in matching
5594 // match(RegP);
5595 format %{ "[$reg]" %}
5596 interface(MEMORY_INTER) %{
5597 base(0x1e); // RSP
5598 index(0x0); // No Index
5599 scale(0x0); // No Scale
5600 disp($reg); // Stack Offset
5601 %}
5602 %}
5603
5604 operand stackSlotI(sRegI reg)
5605 %{
5606 constraint(ALLOC_IN_RC(stack_slots));
5607 // No match rule because this operand is only generated in matching
5608 // match(RegI);
5609 format %{ "[$reg]" %}
5610 interface(MEMORY_INTER) %{
5611 base(0x1e); // RSP
5612 index(0x0); // No Index
5613 scale(0x0); // No Scale
5614 disp($reg); // Stack Offset
5615 %}
5616 %}
5617
5618 operand stackSlotF(sRegF reg)
5619 %{
5620 constraint(ALLOC_IN_RC(stack_slots));
5621 // No match rule because this operand is only generated in matching
5622 // match(RegF);
5623 format %{ "[$reg]" %}
5624 interface(MEMORY_INTER) %{
5625 base(0x1e); // RSP
5626 index(0x0); // No Index
5627 scale(0x0); // No Scale
5628 disp($reg); // Stack Offset
5629 %}
5630 %}
5631
5632 operand stackSlotD(sRegD reg)
5633 %{
5634 constraint(ALLOC_IN_RC(stack_slots));
5635 // No match rule because this operand is only generated in matching
5636 // match(RegD);
5637 format %{ "[$reg]" %}
5638 interface(MEMORY_INTER) %{
5639 base(0x1e); // RSP
5640 index(0x0); // No Index
5641 scale(0x0); // No Scale
5642 disp($reg); // Stack Offset
5643 %}
5644 %}
5645
5646 operand stackSlotL(sRegL reg)
5647 %{
5648 constraint(ALLOC_IN_RC(stack_slots));
5649 // No match rule because this operand is only generated in matching
5650 // match(RegL);
5651 format %{ "[$reg]" %}
5652 interface(MEMORY_INTER) %{
5653 base(0x1e); // RSP
5654 index(0x0); // No Index
5655 scale(0x0); // No Scale
5656 disp($reg); // Stack Offset
5657 %}
5658 %}
5659
5660 // Operands for expressing Control Flow
5661 // NOTE: Label is a predefined operand which should not be redefined in
5662 // the AD file. It is generically handled within the ADLC.
5663
5664 //----------Conditional Branch Operands----------------------------------------
5665 // Comparison Op - This is the operation of the comparison, and is limited to
5666 // the following set of codes:
5667 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5668 //
5669 // Other attributes of the comparison, such as unsignedness, are specified
5670 // by the comparison instruction that sets a condition code flags register.
5671 // That result is represented by a flags operand whose subtype is appropriate
5672 // to the unsignedness (etc.) of the comparison.
5673 //
5674 // Later, the instruction which matches both the Comparison Op (a Bool) and
5675 // the flags (produced by the Cmp) specifies the coding of the comparison op
5676 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5677
5678 // used for signed integral comparisons and fp comparisons
5679
5680 operand cmpOp()
5681 %{
5682 match(Bool);
5683
5684 format %{ "" %}
5685 interface(COND_INTER) %{
5686 equal(0x0, "eq");
5687 not_equal(0x1, "ne");
5688 less(0xb, "lt");
5689 greater_equal(0xa, "ge");
5690 less_equal(0xd, "le");
5691 greater(0xc, "gt");
5692 overflow(0x6, "vs");
5693 no_overflow(0x7, "vc");
5694 %}
5695 %}
5696
5697 // used for unsigned integral comparisons
5698
5699 operand cmpOpU()
5700 %{
5701 match(Bool);
5702
5703 format %{ "" %}
5704 interface(COND_INTER) %{
5705 equal(0x0, "eq");
5706 not_equal(0x1, "ne");
5707 less(0x3, "lo");
5708 greater_equal(0x2, "hs");
5709 less_equal(0x9, "ls");
5710 greater(0x8, "hi");
5711 overflow(0x6, "vs");
5712 no_overflow(0x7, "vc");
5713 %}
5714 %}
5715
5716 // used for certain integral comparisons which can be
5717 // converted to cbxx or tbxx instructions
5718
5719 operand cmpOpEqNe()
5720 %{
5721 match(Bool);
5722 op_cost(0);
5723 predicate(n->as_Bool()->_test._test == BoolTest::ne
5724 || n->as_Bool()->_test._test == BoolTest::eq);
5725
5726 format %{ "" %}
5727 interface(COND_INTER) %{
5728 equal(0x0, "eq");
5729 not_equal(0x1, "ne");
5730 less(0xb, "lt");
5731 greater_equal(0xa, "ge");
5732 less_equal(0xd, "le");
5733 greater(0xc, "gt");
5734 overflow(0x6, "vs");
5735 no_overflow(0x7, "vc");
5736 %}
5737 %}
5738
5739 // used for certain integral comparisons which can be
5740 // converted to cbxx or tbxx instructions
5741
5742 operand cmpOpLtGe()
5743 %{
5744 match(Bool);
5745 op_cost(0);
5746
5747 predicate(n->as_Bool()->_test._test == BoolTest::lt
5748 || n->as_Bool()->_test._test == BoolTest::ge);
5749
5750 format %{ "" %}
5751 interface(COND_INTER) %{
5752 equal(0x0, "eq");
5753 not_equal(0x1, "ne");
5754 less(0xb, "lt");
5755 greater_equal(0xa, "ge");
5756 less_equal(0xd, "le");
5757 greater(0xc, "gt");
5758 overflow(0x6, "vs");
5759 no_overflow(0x7, "vc");
5760 %}
5761 %}
5762
5763 // used for certain unsigned integral comparisons which can be
5764 // converted to cbxx or tbxx instructions
5765
5766 operand cmpOpUEqNeLeGt()
5767 %{
5768 match(Bool);
5769 op_cost(0);
5770
5771 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5772 n->as_Bool()->_test._test == BoolTest::ne ||
5773 n->as_Bool()->_test._test == BoolTest::le ||
5774 n->as_Bool()->_test._test == BoolTest::gt);
5775
5776 format %{ "" %}
5777 interface(COND_INTER) %{
5778 equal(0x0, "eq");
5779 not_equal(0x1, "ne");
5780 less(0x3, "lo");
5781 greater_equal(0x2, "hs");
5782 less_equal(0x9, "ls");
5783 greater(0x8, "hi");
5784 overflow(0x6, "vs");
5785 no_overflow(0x7, "vc");
5786 %}
5787 %}
5788
5789 // Special operand allowing long args to int ops to be truncated for free
5790
5791 operand iRegL2I(iRegL reg) %{
5792
5793 op_cost(0);
5794
5795 match(ConvL2I reg);
5796
5797 format %{ "l2i($reg)" %}
5798
5799 interface(REG_INTER)
5800 %}
5801
5802 operand iRegL2P(iRegL reg) %{
5803
5804 op_cost(0);
5805
5806 match(CastX2P reg);
5807
5808 format %{ "l2p($reg)" %}
5809
5810 interface(REG_INTER)
5811 %}
5812
5813 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5814 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5815 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5816 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5817
5818 //----------OPERAND CLASSES----------------------------------------------------
5819 // Operand Classes are groups of operands that are used as to simplify
5820 // instruction definitions by not requiring the AD writer to specify
5821 // separate instructions for every form of operand when the
5822 // instruction accepts multiple operand types with the same basic
5823 // encoding and format. The classic case of this is memory operands.
5824
5825 // memory is used to define read/write location for load/store
5826 // instruction defs. we can turn a memory op into an Address
5827
5828 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5829 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5830
5831 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5832 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5833
5834 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5835 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5836
5837 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5838 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5839
5840 // All of the memory operands. For the pipeline description.
5841 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5842 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5843 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5844
5845
5846 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5847 // operations. it allows the src to be either an iRegI or a (ConvL2I
5848 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5849 // can be elided because the 32-bit instruction will just employ the
5850 // lower 32 bits anyway.
5851 //
5852 // n.b. this does not elide all L2I conversions. if the truncated
5853 // value is consumed by more than one operation then the ConvL2I
5854 // cannot be bundled into the consuming nodes so an l2i gets planted
5855 // (actually a movw $dst $src) and the downstream instructions consume
5856 // the result of the l2i as an iRegI input. That's a shame since the
5857 // movw is actually redundant but its not too costly.
5858
5859 opclass iRegIorL2I(iRegI, iRegL2I);
5860 opclass iRegPorL2P(iRegP, iRegL2P);
5861
5862 //----------PIPELINE-----------------------------------------------------------
5863 // Rules which define the behavior of the target architectures pipeline.
5864
5865 // For specific pipelines, eg A53, define the stages of that pipeline
5866 //pipe_desc(ISS, EX1, EX2, WR);
5867 #define ISS S0
5868 #define EX1 S1
5869 #define EX2 S2
5870 #define WR S3
5871
5872 // Integer ALU reg operation
5873 pipeline %{
5874
5875 attributes %{
5876 // ARM instructions are of fixed length
5877 fixed_size_instructions; // Fixed size instructions TODO does
5878 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5879 // ARM instructions come in 32-bit word units
5880 instruction_unit_size = 4; // An instruction is 4 bytes long
5881 instruction_fetch_unit_size = 64; // The processor fetches one line
5882 instruction_fetch_units = 1; // of 64 bytes
5883 %}
5884
5885 // We don't use an actual pipeline model so don't care about resources
5886 // or description. we do use pipeline classes to introduce fixed
5887 // latencies
5888
5889 //----------RESOURCES----------------------------------------------------------
5890 // Resources are the functional units available to the machine
5891
5892 resources( INS0, INS1, INS01 = INS0 | INS1,
5893 ALU0, ALU1, ALU = ALU0 | ALU1,
5894 MAC,
5895 DIV,
5896 BRANCH,
5897 LDST,
5898 NEON_FP);
5899
5900 //----------PIPELINE DESCRIPTION-----------------------------------------------
5901 // Pipeline Description specifies the stages in the machine's pipeline
5902
5903 // Define the pipeline as a generic 6 stage pipeline
5904 pipe_desc(S0, S1, S2, S3, S4, S5);
5905
5906 //----------PIPELINE CLASSES---------------------------------------------------
5907 // Pipeline Classes describe the stages in which input and output are
5908 // referenced by the hardware pipeline.
5909
5910 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5911 %{
5912 single_instruction;
5913 src1 : S1(read);
5914 src2 : S2(read);
5915 dst : S5(write);
5916 INS01 : ISS;
5917 NEON_FP : S5;
5918 %}
5919
5920 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5921 %{
5922 single_instruction;
5923 src1 : S1(read);
5924 src2 : S2(read);
5925 dst : S5(write);
5926 INS01 : ISS;
5927 NEON_FP : S5;
5928 %}
5929
5930 pipe_class fp_uop_s(vRegF dst, vRegF src)
5931 %{
5932 single_instruction;
5933 src : S1(read);
5934 dst : S5(write);
5935 INS01 : ISS;
5936 NEON_FP : S5;
5937 %}
5938
5939 pipe_class fp_uop_d(vRegD dst, vRegD src)
5940 %{
5941 single_instruction;
5942 src : S1(read);
5943 dst : S5(write);
5944 INS01 : ISS;
5945 NEON_FP : S5;
5946 %}
5947
5948 pipe_class fp_d2f(vRegF dst, vRegD src)
5949 %{
5950 single_instruction;
5951 src : S1(read);
5952 dst : S5(write);
5953 INS01 : ISS;
5954 NEON_FP : S5;
5955 %}
5956
5957 pipe_class fp_f2d(vRegD dst, vRegF src)
5958 %{
5959 single_instruction;
5960 src : S1(read);
5961 dst : S5(write);
5962 INS01 : ISS;
5963 NEON_FP : S5;
5964 %}
5965
5966 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5967 %{
5968 single_instruction;
5969 src : S1(read);
5970 dst : S5(write);
5971 INS01 : ISS;
5972 NEON_FP : S5;
5973 %}
5974
5975 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5976 %{
5977 single_instruction;
5978 src : S1(read);
5979 dst : S5(write);
5980 INS01 : ISS;
5981 NEON_FP : S5;
5982 %}
5983
5984 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5985 %{
5986 single_instruction;
5987 src : S1(read);
5988 dst : S5(write);
5989 INS01 : ISS;
5990 NEON_FP : S5;
5991 %}
5992
5993 pipe_class fp_l2f(vRegF dst, iRegL src)
5994 %{
5995 single_instruction;
5996 src : S1(read);
5997 dst : S5(write);
5998 INS01 : ISS;
5999 NEON_FP : S5;
6000 %}
6001
6002 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6003 %{
6004 single_instruction;
6005 src : S1(read);
6006 dst : S5(write);
6007 INS01 : ISS;
6008 NEON_FP : S5;
6009 %}
6010
6011 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6012 %{
6013 single_instruction;
6014 src : S1(read);
6015 dst : S5(write);
6016 INS01 : ISS;
6017 NEON_FP : S5;
6018 %}
6019
6020 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6021 %{
6022 single_instruction;
6023 src : S1(read);
6024 dst : S5(write);
6025 INS01 : ISS;
6026 NEON_FP : S5;
6027 %}
6028
6029 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6030 %{
6031 single_instruction;
6032 src : S1(read);
6033 dst : S5(write);
6034 INS01 : ISS;
6035 NEON_FP : S5;
6036 %}
6037
6038 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6039 %{
6040 single_instruction;
6041 src1 : S1(read);
6042 src2 : S2(read);
6043 dst : S5(write);
6044 INS0 : ISS;
6045 NEON_FP : S5;
6046 %}
6047
6048 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6049 %{
6050 single_instruction;
6051 src1 : S1(read);
6052 src2 : S2(read);
6053 dst : S5(write);
6054 INS0 : ISS;
6055 NEON_FP : S5;
6056 %}
6057
6058 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6059 %{
6060 single_instruction;
6061 cr : S1(read);
6062 src1 : S1(read);
6063 src2 : S1(read);
6064 dst : S3(write);
6065 INS01 : ISS;
6066 NEON_FP : S3;
6067 %}
6068
6069 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6070 %{
6071 single_instruction;
6072 cr : S1(read);
6073 src1 : S1(read);
6074 src2 : S1(read);
6075 dst : S3(write);
6076 INS01 : ISS;
6077 NEON_FP : S3;
6078 %}
6079
6080 pipe_class fp_imm_s(vRegF dst)
6081 %{
6082 single_instruction;
6083 dst : S3(write);
6084 INS01 : ISS;
6085 NEON_FP : S3;
6086 %}
6087
6088 pipe_class fp_imm_d(vRegD dst)
6089 %{
6090 single_instruction;
6091 dst : S3(write);
6092 INS01 : ISS;
6093 NEON_FP : S3;
6094 %}
6095
6096 pipe_class fp_load_constant_s(vRegF dst)
6097 %{
6098 single_instruction;
6099 dst : S4(write);
6100 INS01 : ISS;
6101 NEON_FP : S4;
6102 %}
6103
6104 pipe_class fp_load_constant_d(vRegD dst)
6105 %{
6106 single_instruction;
6107 dst : S4(write);
6108 INS01 : ISS;
6109 NEON_FP : S4;
6110 %}
6111
6112 //------- Integer ALU operations --------------------------
6113
6114 // Integer ALU reg-reg operation
6115 // Operands needed in EX1, result generated in EX2
6116 // Eg. ADD x0, x1, x2
6117 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6118 %{
6119 single_instruction;
6120 dst : EX2(write);
6121 src1 : EX1(read);
6122 src2 : EX1(read);
6123 INS01 : ISS; // Dual issue as instruction 0 or 1
6124 ALU : EX2;
6125 %}
6126
6127 // Integer ALU reg-reg operation with constant shift
6128 // Shifted register must be available in LATE_ISS instead of EX1
6129 // Eg. ADD x0, x1, x2, LSL #2
6130 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6131 %{
6132 single_instruction;
6133 dst : EX2(write);
6134 src1 : EX1(read);
6135 src2 : ISS(read);
6136 INS01 : ISS;
6137 ALU : EX2;
6138 %}
6139
6140 // Integer ALU reg operation with constant shift
6141 // Eg. LSL x0, x1, #shift
6142 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6143 %{
6144 single_instruction;
6145 dst : EX2(write);
6146 src1 : ISS(read);
6147 INS01 : ISS;
6148 ALU : EX2;
6149 %}
6150
6151 // Integer ALU reg-reg operation with variable shift
6152 // Both operands must be available in LATE_ISS instead of EX1
6153 // Result is available in EX1 instead of EX2
6154 // Eg. LSLV x0, x1, x2
6155 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6156 %{
6157 single_instruction;
6158 dst : EX1(write);
6159 src1 : ISS(read);
6160 src2 : ISS(read);
6161 INS01 : ISS;
6162 ALU : EX1;
6163 %}
6164
6165 // Integer ALU reg-reg operation with extract
6166 // As for _vshift above, but result generated in EX2
6167 // Eg. EXTR x0, x1, x2, #N
6168 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6169 %{
6170 single_instruction;
6171 dst : EX2(write);
6172 src1 : ISS(read);
6173 src2 : ISS(read);
6174 INS1 : ISS; // Can only dual issue as Instruction 1
6175 ALU : EX1;
6176 %}
6177
6178 // Integer ALU reg operation
6179 // Eg. NEG x0, x1
6180 pipe_class ialu_reg(iRegI dst, iRegI src)
6181 %{
6182 single_instruction;
6183 dst : EX2(write);
6184 src : EX1(read);
6185 INS01 : ISS;
6186 ALU : EX2;
6187 %}
6188
6189 // Integer ALU reg mmediate operation
6190 // Eg. ADD x0, x1, #N
6191 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6192 %{
6193 single_instruction;
6194 dst : EX2(write);
6195 src1 : EX1(read);
6196 INS01 : ISS;
6197 ALU : EX2;
6198 %}
6199
6200 // Integer ALU immediate operation (no source operands)
6201 // Eg. MOV x0, #N
6202 pipe_class ialu_imm(iRegI dst)
6203 %{
6204 single_instruction;
6205 dst : EX1(write);
6206 INS01 : ISS;
6207 ALU : EX1;
6208 %}
6209
6210 //------- Compare operation -------------------------------
6211
6212 // Compare reg-reg
6213 // Eg. CMP x0, x1
6214 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6215 %{
6216 single_instruction;
6217 // fixed_latency(16);
6218 cr : EX2(write);
6219 op1 : EX1(read);
6220 op2 : EX1(read);
6221 INS01 : ISS;
6222 ALU : EX2;
6223 %}
6224
6225 // Compare reg-reg
6226 // Eg. CMP x0, #N
6227 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6228 %{
6229 single_instruction;
6230 // fixed_latency(16);
6231 cr : EX2(write);
6232 op1 : EX1(read);
6233 INS01 : ISS;
6234 ALU : EX2;
6235 %}
6236
6237 //------- Conditional instructions ------------------------
6238
6239 // Conditional no operands
6240 // Eg. CSINC x0, zr, zr, <cond>
6241 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6242 %{
6243 single_instruction;
6244 cr : EX1(read);
6245 dst : EX2(write);
6246 INS01 : ISS;
6247 ALU : EX2;
6248 %}
6249
6250 // Conditional 2 operand
6251 // EG. CSEL X0, X1, X2, <cond>
6252 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6253 %{
6254 single_instruction;
6255 cr : EX1(read);
6256 src1 : EX1(read);
6257 src2 : EX1(read);
6258 dst : EX2(write);
6259 INS01 : ISS;
6260 ALU : EX2;
6261 %}
6262
6263 // Conditional 2 operand
6264 // EG. CSEL X0, X1, X2, <cond>
6265 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6266 %{
6267 single_instruction;
6268 cr : EX1(read);
6269 src : EX1(read);
6270 dst : EX2(write);
6271 INS01 : ISS;
6272 ALU : EX2;
6273 %}
6274
6275 //------- Multiply pipeline operations --------------------
6276
6277 // Multiply reg-reg
6278 // Eg. MUL w0, w1, w2
6279 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6280 %{
6281 single_instruction;
6282 dst : WR(write);
6283 src1 : ISS(read);
6284 src2 : ISS(read);
6285 INS01 : ISS;
6286 MAC : WR;
6287 %}
6288
6289 // Multiply accumulate
6290 // Eg. MADD w0, w1, w2, w3
6291 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6292 %{
6293 single_instruction;
6294 dst : WR(write);
6295 src1 : ISS(read);
6296 src2 : ISS(read);
6297 src3 : ISS(read);
6298 INS01 : ISS;
6299 MAC : WR;
6300 %}
6301
6302 // Eg. MUL w0, w1, w2
6303 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6304 %{
6305 single_instruction;
6306 fixed_latency(3); // Maximum latency for 64 bit mul
6307 dst : WR(write);
6308 src1 : ISS(read);
6309 src2 : ISS(read);
6310 INS01 : ISS;
6311 MAC : WR;
6312 %}
6313
6314 // Multiply accumulate
6315 // Eg. MADD w0, w1, w2, w3
6316 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6317 %{
6318 single_instruction;
6319 fixed_latency(3); // Maximum latency for 64 bit mul
6320 dst : WR(write);
6321 src1 : ISS(read);
6322 src2 : ISS(read);
6323 src3 : ISS(read);
6324 INS01 : ISS;
6325 MAC : WR;
6326 %}
6327
6328 //------- Divide pipeline operations --------------------
6329
6330 // Eg. SDIV w0, w1, w2
6331 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6332 %{
6333 single_instruction;
6334 fixed_latency(8); // Maximum latency for 32 bit divide
6335 dst : WR(write);
6336 src1 : ISS(read);
6337 src2 : ISS(read);
6338 INS0 : ISS; // Can only dual issue as instruction 0
6339 DIV : WR;
6340 %}
6341
6342 // Eg. SDIV x0, x1, x2
6343 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6344 %{
6345 single_instruction;
6346 fixed_latency(16); // Maximum latency for 64 bit divide
6347 dst : WR(write);
6348 src1 : ISS(read);
6349 src2 : ISS(read);
6350 INS0 : ISS; // Can only dual issue as instruction 0
6351 DIV : WR;
6352 %}
6353
6354 //------- Load pipeline operations ------------------------
6355
6356 // Load - prefetch
6357 // Eg. PFRM <mem>
6358 pipe_class iload_prefetch(memory mem)
6359 %{
6360 single_instruction;
6361 mem : ISS(read);
6362 INS01 : ISS;
6363 LDST : WR;
6364 %}
6365
6366 // Load - reg, mem
6367 // Eg. LDR x0, <mem>
6368 pipe_class iload_reg_mem(iRegI dst, memory mem)
6369 %{
6370 single_instruction;
6371 dst : WR(write);
6372 mem : ISS(read);
6373 INS01 : ISS;
6374 LDST : WR;
6375 %}
6376
6377 // Load - reg, reg
6378 // Eg. LDR x0, [sp, x1]
6379 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6380 %{
6381 single_instruction;
6382 dst : WR(write);
6383 src : ISS(read);
6384 INS01 : ISS;
6385 LDST : WR;
6386 %}
6387
6388 //------- Store pipeline operations -----------------------
6389
6390 // Store - zr, mem
6391 // Eg. STR zr, <mem>
6392 pipe_class istore_mem(memory mem)
6393 %{
6394 single_instruction;
6395 mem : ISS(read);
6396 INS01 : ISS;
6397 LDST : WR;
6398 %}
6399
6400 // Store - reg, mem
6401 // Eg. STR x0, <mem>
6402 pipe_class istore_reg_mem(iRegI src, memory mem)
6403 %{
6404 single_instruction;
6405 mem : ISS(read);
6406 src : EX2(read);
6407 INS01 : ISS;
6408 LDST : WR;
6409 %}
6410
6411 // Store - reg, reg
6412 // Eg. STR x0, [sp, x1]
6413 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6414 %{
6415 single_instruction;
6416 dst : ISS(read);
6417 src : EX2(read);
6418 INS01 : ISS;
6419 LDST : WR;
6420 %}
6421
6422 //------- Store pipeline operations -----------------------
6423
6424 // Branch
6425 pipe_class pipe_branch()
6426 %{
6427 single_instruction;
6428 INS01 : ISS;
6429 BRANCH : EX1;
6430 %}
6431
6432 // Conditional branch
6433 pipe_class pipe_branch_cond(rFlagsReg cr)
6434 %{
6435 single_instruction;
6436 cr : EX1(read);
6437 INS01 : ISS;
6438 BRANCH : EX1;
6439 %}
6440
6441 // Compare & Branch
6442 // EG. CBZ/CBNZ
6443 pipe_class pipe_cmp_branch(iRegI op1)
6444 %{
6445 single_instruction;
6446 op1 : EX1(read);
6447 INS01 : ISS;
6448 BRANCH : EX1;
6449 %}
6450
6451 //------- Synchronisation operations ----------------------
6452
6453 // Any operation requiring serialization.
6454 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6455 pipe_class pipe_serial()
6456 %{
6457 single_instruction;
6458 force_serialization;
6459 fixed_latency(16);
6460 INS01 : ISS(2); // Cannot dual issue with any other instruction
6461 LDST : WR;
6462 %}
6463
6464 // Generic big/slow expanded idiom - also serialized
6465 pipe_class pipe_slow()
6466 %{
6467 instruction_count(10);
6468 multiple_bundles;
6469 force_serialization;
6470 fixed_latency(16);
6471 INS01 : ISS(2); // Cannot dual issue with any other instruction
6472 LDST : WR;
6473 %}
6474
6475 // Empty pipeline class
6476 pipe_class pipe_class_empty()
6477 %{
6478 single_instruction;
6479 fixed_latency(0);
6480 %}
6481
6482 // Default pipeline class.
6483 pipe_class pipe_class_default()
6484 %{
6485 single_instruction;
6486 fixed_latency(2);
6487 %}
6488
6489 // Pipeline class for compares.
6490 pipe_class pipe_class_compare()
6491 %{
6492 single_instruction;
6493 fixed_latency(16);
6494 %}
6495
6496 // Pipeline class for memory operations.
6497 pipe_class pipe_class_memory()
6498 %{
6499 single_instruction;
6500 fixed_latency(16);
6501 %}
6502
6503 // Pipeline class for call.
6504 pipe_class pipe_class_call()
6505 %{
6506 single_instruction;
6507 fixed_latency(100);
6508 %}
6509
6510 // Define the class for the Nop node.
6511 define %{
6512 MachNop = pipe_class_empty;
6513 %}
6514
6515 %}
6516 //----------INSTRUCTIONS-------------------------------------------------------
6517 //
6518 // match -- States which machine-independent subtree may be replaced
6519 // by this instruction.
6520 // ins_cost -- The estimated cost of this instruction is used by instruction
6521 // selection to identify a minimum cost tree of machine
6522 // instructions that matches a tree of machine-independent
6523 // instructions.
6524 // format -- A string providing the disassembly for this instruction.
6525 // The value of an instruction's operand may be inserted
6526 // by referring to it with a '$' prefix.
6527 // opcode -- Three instruction opcodes may be provided. These are referred
6528 // to within an encode class as $primary, $secondary, and $tertiary
6529 // rrspectively. The primary opcode is commonly used to
6530 // indicate the type of machine instruction, while secondary
6531 // and tertiary are often used for prefix options or addressing
6532 // modes.
6533 // ins_encode -- A list of encode classes with parameters. The encode class
6534 // name must have been defined in an 'enc_class' specification
6535 // in the encode section of the architecture description.
6536
6537 // ============================================================================
6538 // Memory (Load/Store) Instructions
6539
6540 // Load Instructions
6541
6542 // Load Byte (8 bit signed)
6543 instruct loadB(iRegINoSp dst, memory1 mem)
6544 %{
6545 match(Set dst (LoadB mem));
6546 predicate(!needs_acquiring_load(n));
6547
6548 ins_cost(4 * INSN_COST);
6549 format %{ "ldrsbw $dst, $mem\t# byte" %}
6550
6551 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6552
6553 ins_pipe(iload_reg_mem);
6554 %}
6555
6556 // Load Byte (8 bit signed) into long
6557 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6558 %{
6559 match(Set dst (ConvI2L (LoadB mem)));
6560 predicate(!needs_acquiring_load(n->in(1)));
6561
6562 ins_cost(4 * INSN_COST);
6563 format %{ "ldrsb $dst, $mem\t# byte" %}
6564
6565 ins_encode(aarch64_enc_ldrsb(dst, mem));
6566
6567 ins_pipe(iload_reg_mem);
6568 %}
6569
6570 // Load Byte (8 bit unsigned)
6571 instruct loadUB(iRegINoSp dst, memory1 mem)
6572 %{
6573 match(Set dst (LoadUB mem));
6574 predicate(!needs_acquiring_load(n));
6575
6576 ins_cost(4 * INSN_COST);
6577 format %{ "ldrbw $dst, $mem\t# byte" %}
6578
6579 ins_encode(aarch64_enc_ldrb(dst, mem));
6580
6581 ins_pipe(iload_reg_mem);
6582 %}
6583
6584 // Load Byte (8 bit unsigned) into long
6585 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6586 %{
6587 match(Set dst (ConvI2L (LoadUB mem)));
6588 predicate(!needs_acquiring_load(n->in(1)));
6589
6590 ins_cost(4 * INSN_COST);
6591 format %{ "ldrb $dst, $mem\t# byte" %}
6592
6593 ins_encode(aarch64_enc_ldrb(dst, mem));
6594
6595 ins_pipe(iload_reg_mem);
6596 %}
6597
6598 // Load Short (16 bit signed)
6599 instruct loadS(iRegINoSp dst, memory2 mem)
6600 %{
6601 match(Set dst (LoadS mem));
6602 predicate(!needs_acquiring_load(n));
6603
6604 ins_cost(4 * INSN_COST);
6605 format %{ "ldrshw $dst, $mem\t# short" %}
6606
6607 ins_encode(aarch64_enc_ldrshw(dst, mem));
6608
6609 ins_pipe(iload_reg_mem);
6610 %}
6611
6612 // Load Short (16 bit signed) into long
6613 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6614 %{
6615 match(Set dst (ConvI2L (LoadS mem)));
6616 predicate(!needs_acquiring_load(n->in(1)));
6617
6618 ins_cost(4 * INSN_COST);
6619 format %{ "ldrsh $dst, $mem\t# short" %}
6620
6621 ins_encode(aarch64_enc_ldrsh(dst, mem));
6622
6623 ins_pipe(iload_reg_mem);
6624 %}
6625
6626 // Load Char (16 bit unsigned)
6627 instruct loadUS(iRegINoSp dst, memory2 mem)
6628 %{
6629 match(Set dst (LoadUS mem));
6630 predicate(!needs_acquiring_load(n));
6631
6632 ins_cost(4 * INSN_COST);
6633 format %{ "ldrh $dst, $mem\t# short" %}
6634
6635 ins_encode(aarch64_enc_ldrh(dst, mem));
6636
6637 ins_pipe(iload_reg_mem);
6638 %}
6639
6640 // Load Short/Char (16 bit unsigned) into long
6641 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6642 %{
6643 match(Set dst (ConvI2L (LoadUS mem)));
6644 predicate(!needs_acquiring_load(n->in(1)));
6645
6646 ins_cost(4 * INSN_COST);
6647 format %{ "ldrh $dst, $mem\t# short" %}
6648
6649 ins_encode(aarch64_enc_ldrh(dst, mem));
6650
6651 ins_pipe(iload_reg_mem);
6652 %}
6653
6654 // Load Integer (32 bit signed)
6655 instruct loadI(iRegINoSp dst, memory4 mem)
6656 %{
6657 match(Set dst (LoadI mem));
6658 predicate(!needs_acquiring_load(n));
6659
6660 ins_cost(4 * INSN_COST);
6661 format %{ "ldrw $dst, $mem\t# int" %}
6662
6663 ins_encode(aarch64_enc_ldrw(dst, mem));
6664
6665 ins_pipe(iload_reg_mem);
6666 %}
6667
6668 // Load Integer (32 bit signed) into long
6669 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6670 %{
6671 match(Set dst (ConvI2L (LoadI mem)));
6672 predicate(!needs_acquiring_load(n->in(1)));
6673
6674 ins_cost(4 * INSN_COST);
6675 format %{ "ldrsw $dst, $mem\t# int" %}
6676
6677 ins_encode(aarch64_enc_ldrsw(dst, mem));
6678
6679 ins_pipe(iload_reg_mem);
6680 %}
6681
6682 // Load Integer (32 bit unsigned) into long
6683 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6684 %{
6685 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6686 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6687
6688 ins_cost(4 * INSN_COST);
6689 format %{ "ldrw $dst, $mem\t# int" %}
6690
6691 ins_encode(aarch64_enc_ldrw(dst, mem));
6692
6693 ins_pipe(iload_reg_mem);
6694 %}
6695
6696 // Load Long (64 bit signed)
6697 instruct loadL(iRegLNoSp dst, memory8 mem)
6698 %{
6699 match(Set dst (LoadL mem));
6700 predicate(!needs_acquiring_load(n));
6701
6702 ins_cost(4 * INSN_COST);
6703 format %{ "ldr $dst, $mem\t# int" %}
6704
6705 ins_encode(aarch64_enc_ldr(dst, mem));
6706
6707 ins_pipe(iload_reg_mem);
6708 %}
6709
6710 // Load Range
6711 instruct loadRange(iRegINoSp dst, memory4 mem)
6712 %{
6713 match(Set dst (LoadRange mem));
6714
6715 ins_cost(4 * INSN_COST);
6716 format %{ "ldrw $dst, $mem\t# range" %}
6717
6718 ins_encode(aarch64_enc_ldrw(dst, mem));
6719
6720 ins_pipe(iload_reg_mem);
6721 %}
6722
6723 // Load Pointer
6724 instruct loadP(iRegPNoSp dst, memory8 mem)
6725 %{
6726 match(Set dst (LoadP mem));
6727 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6728
6729 ins_cost(4 * INSN_COST);
6730 format %{ "ldr $dst, $mem\t# ptr" %}
6731
6732 ins_encode(aarch64_enc_ldr(dst, mem));
6733
6734 ins_pipe(iload_reg_mem);
6735 %}
6736
6737 // Load Compressed Pointer
6738 instruct loadN(iRegNNoSp dst, memory4 mem)
6739 %{
6740 match(Set dst (LoadN mem));
6741 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6742
6743 ins_cost(4 * INSN_COST);
6744 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6745
6746 ins_encode(aarch64_enc_ldrw(dst, mem));
6747
6748 ins_pipe(iload_reg_mem);
6749 %}
6750
6751 // Load Klass Pointer
6752 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6753 %{
6754 match(Set dst (LoadKlass mem));
6755 predicate(!needs_acquiring_load(n));
6756
6757 ins_cost(4 * INSN_COST);
6758 format %{ "ldr $dst, $mem\t# class" %}
6759
6760 ins_encode(aarch64_enc_ldr(dst, mem));
6761
6762 ins_pipe(iload_reg_mem);
6763 %}
6764
6765 // Load Narrow Klass Pointer
6766 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6767 %{
6768 match(Set dst (LoadNKlass mem));
6769 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6770
6771 ins_cost(4 * INSN_COST);
6772 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6773
6774 ins_encode(aarch64_enc_ldrw(dst, mem));
6775
6776 ins_pipe(iload_reg_mem);
6777 %}
6778
6779 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6780 %{
6781 match(Set dst (LoadNKlass mem));
6782 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6783
6784 ins_cost(4 * INSN_COST);
6785 format %{
6786 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6787 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6788 %}
6789 ins_encode %{
6790 // inlined aarch64_enc_ldrw
6791 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6792 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6793 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6794 %}
6795 ins_pipe(iload_reg_mem);
6796 %}
6797
6798 // Load Float
6799 instruct loadF(vRegF dst, memory4 mem)
6800 %{
6801 match(Set dst (LoadF mem));
6802 predicate(!needs_acquiring_load(n));
6803
6804 ins_cost(4 * INSN_COST);
6805 format %{ "ldrs $dst, $mem\t# float" %}
6806
6807 ins_encode( aarch64_enc_ldrs(dst, mem) );
6808
6809 ins_pipe(pipe_class_memory);
6810 %}
6811
6812 // Load Double
6813 instruct loadD(vRegD dst, memory8 mem)
6814 %{
6815 match(Set dst (LoadD mem));
6816 predicate(!needs_acquiring_load(n));
6817
6818 ins_cost(4 * INSN_COST);
6819 format %{ "ldrd $dst, $mem\t# double" %}
6820
6821 ins_encode( aarch64_enc_ldrd(dst, mem) );
6822
6823 ins_pipe(pipe_class_memory);
6824 %}
6825
6826
6827 // Load Int Constant
6828 instruct loadConI(iRegINoSp dst, immI src)
6829 %{
6830 match(Set dst src);
6831
6832 ins_cost(INSN_COST);
6833 format %{ "mov $dst, $src\t# int" %}
6834
6835 ins_encode( aarch64_enc_movw_imm(dst, src) );
6836
6837 ins_pipe(ialu_imm);
6838 %}
6839
6840 // Load Long Constant
6841 instruct loadConL(iRegLNoSp dst, immL src)
6842 %{
6843 match(Set dst src);
6844
6845 ins_cost(INSN_COST);
6846 format %{ "mov $dst, $src\t# long" %}
6847
6848 ins_encode( aarch64_enc_mov_imm(dst, src) );
6849
6850 ins_pipe(ialu_imm);
6851 %}
6852
6853 // Load Pointer Constant
6854
6855 instruct loadConP(iRegPNoSp dst, immP con)
6856 %{
6857 match(Set dst con);
6858
6859 ins_cost(INSN_COST * 4);
6860 format %{
6861 "mov $dst, $con\t# ptr\n\t"
6862 %}
6863
6864 ins_encode(aarch64_enc_mov_p(dst, con));
6865
6866 ins_pipe(ialu_imm);
6867 %}
6868
6869 // Load Null Pointer Constant
6870
6871 instruct loadConP0(iRegPNoSp dst, immP0 con)
6872 %{
6873 match(Set dst con);
6874
6875 ins_cost(INSN_COST);
6876 format %{ "mov $dst, $con\t# nullptr ptr" %}
6877
6878 ins_encode(aarch64_enc_mov_p0(dst, con));
6879
6880 ins_pipe(ialu_imm);
6881 %}
6882
6883 // Load Pointer Constant One
6884
6885 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6886 %{
6887 match(Set dst con);
6888
6889 ins_cost(INSN_COST);
6890 format %{ "mov $dst, $con\t# nullptr ptr" %}
6891
6892 ins_encode(aarch64_enc_mov_p1(dst, con));
6893
6894 ins_pipe(ialu_imm);
6895 %}
6896
6897 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6898 %{
6899 match(Set dst con);
6900
6901 ins_cost(INSN_COST);
6902 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6903
6904 ins_encode %{
6905 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6906 %}
6907
6908 ins_pipe(ialu_imm);
6909 %}
6910
6911 // Load Narrow Pointer Constant
6912
6913 instruct loadConN(iRegNNoSp dst, immN con)
6914 %{
6915 match(Set dst con);
6916
6917 ins_cost(INSN_COST * 4);
6918 format %{ "mov $dst, $con\t# compressed ptr" %}
6919
6920 ins_encode(aarch64_enc_mov_n(dst, con));
6921
6922 ins_pipe(ialu_imm);
6923 %}
6924
6925 // Load Narrow Null Pointer Constant
6926
6927 instruct loadConN0(iRegNNoSp dst, immN0 con)
6928 %{
6929 match(Set dst con);
6930
6931 ins_cost(INSN_COST);
6932 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6933
6934 ins_encode(aarch64_enc_mov_n0(dst, con));
6935
6936 ins_pipe(ialu_imm);
6937 %}
6938
6939 // Load Narrow Klass Constant
6940
6941 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6942 %{
6943 match(Set dst con);
6944
6945 ins_cost(INSN_COST);
6946 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6947
6948 ins_encode(aarch64_enc_mov_nk(dst, con));
6949
6950 ins_pipe(ialu_imm);
6951 %}
6952
6953 // Load Packed Float Constant
6954
6955 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6956 match(Set dst con);
6957 ins_cost(INSN_COST * 4);
6958 format %{ "fmovs $dst, $con"%}
6959 ins_encode %{
6960 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6961 %}
6962
6963 ins_pipe(fp_imm_s);
6964 %}
6965
6966 // Load Float Constant
6967
6968 instruct loadConF(vRegF dst, immF con) %{
6969 match(Set dst con);
6970
6971 ins_cost(INSN_COST * 4);
6972
6973 format %{
6974 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6975 %}
6976
6977 ins_encode %{
6978 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6979 %}
6980
6981 ins_pipe(fp_load_constant_s);
6982 %}
6983
6984 // Load Packed Double Constant
6985
6986 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6987 match(Set dst con);
6988 ins_cost(INSN_COST);
6989 format %{ "fmovd $dst, $con"%}
6990 ins_encode %{
6991 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6992 %}
6993
6994 ins_pipe(fp_imm_d);
6995 %}
6996
6997 // Load Double Constant
6998
6999 instruct loadConD(vRegD dst, immD con) %{
7000 match(Set dst con);
7001
7002 ins_cost(INSN_COST * 5);
7003 format %{
7004 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7005 %}
7006
7007 ins_encode %{
7008 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7009 %}
7010
7011 ins_pipe(fp_load_constant_d);
7012 %}
7013
7014 // Load Half Float Constant
7015 instruct loadConH(vRegF dst, immH con) %{
7016 match(Set dst con);
7017 format %{ "mov rscratch1, $con\n\t"
7018 "fmov $dst, rscratch1"
7019 %}
7020 ins_encode %{
7021 __ movw(rscratch1, (uint32_t)$con$$constant);
7022 __ fmovs($dst$$FloatRegister, rscratch1);
7023 %}
7024 ins_pipe(pipe_class_default);
7025 %}
7026
7027 // Store Instructions
7028
7029 // Store Byte
7030 instruct storeB(iRegIorL2I src, memory1 mem)
7031 %{
7032 match(Set mem (StoreB mem src));
7033 predicate(!needs_releasing_store(n));
7034
7035 ins_cost(INSN_COST);
7036 format %{ "strb $src, $mem\t# byte" %}
7037
7038 ins_encode(aarch64_enc_strb(src, mem));
7039
7040 ins_pipe(istore_reg_mem);
7041 %}
7042
7043
7044 instruct storeimmB0(immI0 zero, memory1 mem)
7045 %{
7046 match(Set mem (StoreB mem zero));
7047 predicate(!needs_releasing_store(n));
7048
7049 ins_cost(INSN_COST);
7050 format %{ "strb rscractch2, $mem\t# byte" %}
7051
7052 ins_encode(aarch64_enc_strb0(mem));
7053
7054 ins_pipe(istore_mem);
7055 %}
7056
7057 // Store Char/Short
7058 instruct storeC(iRegIorL2I src, memory2 mem)
7059 %{
7060 match(Set mem (StoreC mem src));
7061 predicate(!needs_releasing_store(n));
7062
7063 ins_cost(INSN_COST);
7064 format %{ "strh $src, $mem\t# short" %}
7065
7066 ins_encode(aarch64_enc_strh(src, mem));
7067
7068 ins_pipe(istore_reg_mem);
7069 %}
7070
7071 instruct storeimmC0(immI0 zero, memory2 mem)
7072 %{
7073 match(Set mem (StoreC mem zero));
7074 predicate(!needs_releasing_store(n));
7075
7076 ins_cost(INSN_COST);
7077 format %{ "strh zr, $mem\t# short" %}
7078
7079 ins_encode(aarch64_enc_strh0(mem));
7080
7081 ins_pipe(istore_mem);
7082 %}
7083
7084 // Store Integer
7085
7086 instruct storeI(iRegIorL2I src, memory4 mem)
7087 %{
7088 match(Set mem(StoreI mem src));
7089 predicate(!needs_releasing_store(n));
7090
7091 ins_cost(INSN_COST);
7092 format %{ "strw $src, $mem\t# int" %}
7093
7094 ins_encode(aarch64_enc_strw(src, mem));
7095
7096 ins_pipe(istore_reg_mem);
7097 %}
7098
7099 instruct storeimmI0(immI0 zero, memory4 mem)
7100 %{
7101 match(Set mem(StoreI mem zero));
7102 predicate(!needs_releasing_store(n));
7103
7104 ins_cost(INSN_COST);
7105 format %{ "strw zr, $mem\t# int" %}
7106
7107 ins_encode(aarch64_enc_strw0(mem));
7108
7109 ins_pipe(istore_mem);
7110 %}
7111
7112 // Store Long (64 bit signed)
7113 instruct storeL(iRegL src, memory8 mem)
7114 %{
7115 match(Set mem (StoreL mem src));
7116 predicate(!needs_releasing_store(n));
7117
7118 ins_cost(INSN_COST);
7119 format %{ "str $src, $mem\t# int" %}
7120
7121 ins_encode(aarch64_enc_str(src, mem));
7122
7123 ins_pipe(istore_reg_mem);
7124 %}
7125
7126 // Store Long (64 bit signed)
7127 instruct storeimmL0(immL0 zero, memory8 mem)
7128 %{
7129 match(Set mem (StoreL mem zero));
7130 predicate(!needs_releasing_store(n));
7131
7132 ins_cost(INSN_COST);
7133 format %{ "str zr, $mem\t# int" %}
7134
7135 ins_encode(aarch64_enc_str0(mem));
7136
7137 ins_pipe(istore_mem);
7138 %}
7139
7140 // Store Pointer
7141 instruct storeP(iRegP src, memory8 mem)
7142 %{
7143 match(Set mem (StoreP mem src));
7144 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7145
7146 ins_cost(INSN_COST);
7147 format %{ "str $src, $mem\t# ptr" %}
7148
7149 ins_encode(aarch64_enc_str(src, mem));
7150
7151 ins_pipe(istore_reg_mem);
7152 %}
7153
7154 // Store Pointer
7155 instruct storeimmP0(immP0 zero, memory8 mem)
7156 %{
7157 match(Set mem (StoreP mem zero));
7158 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7159
7160 ins_cost(INSN_COST);
7161 format %{ "str zr, $mem\t# ptr" %}
7162
7163 ins_encode(aarch64_enc_str0(mem));
7164
7165 ins_pipe(istore_mem);
7166 %}
7167
7168 // Store Compressed Pointer
7169 instruct storeN(iRegN src, memory4 mem)
7170 %{
7171 match(Set mem (StoreN mem src));
7172 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7173
7174 ins_cost(INSN_COST);
7175 format %{ "strw $src, $mem\t# compressed ptr" %}
7176
7177 ins_encode(aarch64_enc_strw(src, mem));
7178
7179 ins_pipe(istore_reg_mem);
7180 %}
7181
7182 instruct storeImmN0(immN0 zero, memory4 mem)
7183 %{
7184 match(Set mem (StoreN mem zero));
7185 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7186
7187 ins_cost(INSN_COST);
7188 format %{ "strw zr, $mem\t# compressed ptr" %}
7189
7190 ins_encode(aarch64_enc_strw0(mem));
7191
7192 ins_pipe(istore_mem);
7193 %}
7194
7195 // Store Float
7196 instruct storeF(vRegF src, memory4 mem)
7197 %{
7198 match(Set mem (StoreF mem src));
7199 predicate(!needs_releasing_store(n));
7200
7201 ins_cost(INSN_COST);
7202 format %{ "strs $src, $mem\t# float" %}
7203
7204 ins_encode( aarch64_enc_strs(src, mem) );
7205
7206 ins_pipe(pipe_class_memory);
7207 %}
7208
7209 // TODO
7210 // implement storeImmF0 and storeFImmPacked
7211
7212 // Store Double
7213 instruct storeD(vRegD src, memory8 mem)
7214 %{
7215 match(Set mem (StoreD mem src));
7216 predicate(!needs_releasing_store(n));
7217
7218 ins_cost(INSN_COST);
7219 format %{ "strd $src, $mem\t# double" %}
7220
7221 ins_encode( aarch64_enc_strd(src, mem) );
7222
7223 ins_pipe(pipe_class_memory);
7224 %}
7225
7226 // Store Compressed Klass Pointer
7227 instruct storeNKlass(iRegN src, memory4 mem)
7228 %{
7229 predicate(!needs_releasing_store(n));
7230 match(Set mem (StoreNKlass mem src));
7231
7232 ins_cost(INSN_COST);
7233 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7234
7235 ins_encode(aarch64_enc_strw(src, mem));
7236
7237 ins_pipe(istore_reg_mem);
7238 %}
7239
7240 // TODO
7241 // implement storeImmD0 and storeDImmPacked
7242
7243 // prefetch instructions
7244 // Must be safe to execute with invalid address (cannot fault).
7245
7246 instruct prefetchalloc( memory8 mem ) %{
7247 match(PrefetchAllocation mem);
7248
7249 ins_cost(INSN_COST);
7250 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7251
7252 ins_encode( aarch64_enc_prefetchw(mem) );
7253
7254 ins_pipe(iload_prefetch);
7255 %}
7256
7257 // ---------------- volatile loads and stores ----------------
7258
7259 // Load Byte (8 bit signed)
7260 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7261 %{
7262 match(Set dst (LoadB mem));
7263
7264 ins_cost(VOLATILE_REF_COST);
7265 format %{ "ldarsb $dst, $mem\t# byte" %}
7266
7267 ins_encode(aarch64_enc_ldarsb(dst, mem));
7268
7269 ins_pipe(pipe_serial);
7270 %}
7271
7272 // Load Byte (8 bit signed) into long
7273 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7274 %{
7275 match(Set dst (ConvI2L (LoadB mem)));
7276
7277 ins_cost(VOLATILE_REF_COST);
7278 format %{ "ldarsb $dst, $mem\t# byte" %}
7279
7280 ins_encode(aarch64_enc_ldarsb(dst, mem));
7281
7282 ins_pipe(pipe_serial);
7283 %}
7284
7285 // Load Byte (8 bit unsigned)
7286 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7287 %{
7288 match(Set dst (LoadUB mem));
7289
7290 ins_cost(VOLATILE_REF_COST);
7291 format %{ "ldarb $dst, $mem\t# byte" %}
7292
7293 ins_encode(aarch64_enc_ldarb(dst, mem));
7294
7295 ins_pipe(pipe_serial);
7296 %}
7297
7298 // Load Byte (8 bit unsigned) into long
7299 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7300 %{
7301 match(Set dst (ConvI2L (LoadUB mem)));
7302
7303 ins_cost(VOLATILE_REF_COST);
7304 format %{ "ldarb $dst, $mem\t# byte" %}
7305
7306 ins_encode(aarch64_enc_ldarb(dst, mem));
7307
7308 ins_pipe(pipe_serial);
7309 %}
7310
7311 // Load Short (16 bit signed)
7312 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7313 %{
7314 match(Set dst (LoadS mem));
7315
7316 ins_cost(VOLATILE_REF_COST);
7317 format %{ "ldarshw $dst, $mem\t# short" %}
7318
7319 ins_encode(aarch64_enc_ldarshw(dst, mem));
7320
7321 ins_pipe(pipe_serial);
7322 %}
7323
7324 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7325 %{
7326 match(Set dst (LoadUS mem));
7327
7328 ins_cost(VOLATILE_REF_COST);
7329 format %{ "ldarhw $dst, $mem\t# short" %}
7330
7331 ins_encode(aarch64_enc_ldarhw(dst, mem));
7332
7333 ins_pipe(pipe_serial);
7334 %}
7335
7336 // Load Short/Char (16 bit unsigned) into long
7337 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7338 %{
7339 match(Set dst (ConvI2L (LoadUS mem)));
7340
7341 ins_cost(VOLATILE_REF_COST);
7342 format %{ "ldarh $dst, $mem\t# short" %}
7343
7344 ins_encode(aarch64_enc_ldarh(dst, mem));
7345
7346 ins_pipe(pipe_serial);
7347 %}
7348
7349 // Load Short/Char (16 bit signed) into long
7350 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7351 %{
7352 match(Set dst (ConvI2L (LoadS mem)));
7353
7354 ins_cost(VOLATILE_REF_COST);
7355 format %{ "ldarh $dst, $mem\t# short" %}
7356
7357 ins_encode(aarch64_enc_ldarsh(dst, mem));
7358
7359 ins_pipe(pipe_serial);
7360 %}
7361
7362 // Load Integer (32 bit signed)
7363 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7364 %{
7365 match(Set dst (LoadI mem));
7366
7367 ins_cost(VOLATILE_REF_COST);
7368 format %{ "ldarw $dst, $mem\t# int" %}
7369
7370 ins_encode(aarch64_enc_ldarw(dst, mem));
7371
7372 ins_pipe(pipe_serial);
7373 %}
7374
7375 // Load Integer (32 bit unsigned) into long
7376 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7377 %{
7378 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7379
7380 ins_cost(VOLATILE_REF_COST);
7381 format %{ "ldarw $dst, $mem\t# int" %}
7382
7383 ins_encode(aarch64_enc_ldarw(dst, mem));
7384
7385 ins_pipe(pipe_serial);
7386 %}
7387
7388 // Load Long (64 bit signed)
7389 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7390 %{
7391 match(Set dst (LoadL mem));
7392
7393 ins_cost(VOLATILE_REF_COST);
7394 format %{ "ldar $dst, $mem\t# int" %}
7395
7396 ins_encode(aarch64_enc_ldar(dst, mem));
7397
7398 ins_pipe(pipe_serial);
7399 %}
7400
7401 // Load Pointer
7402 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7403 %{
7404 match(Set dst (LoadP mem));
7405 predicate(n->as_Load()->barrier_data() == 0);
7406
7407 ins_cost(VOLATILE_REF_COST);
7408 format %{ "ldar $dst, $mem\t# ptr" %}
7409
7410 ins_encode(aarch64_enc_ldar(dst, mem));
7411
7412 ins_pipe(pipe_serial);
7413 %}
7414
7415 // Load Compressed Pointer
7416 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7417 %{
7418 match(Set dst (LoadN mem));
7419 predicate(n->as_Load()->barrier_data() == 0);
7420
7421 ins_cost(VOLATILE_REF_COST);
7422 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7423
7424 ins_encode(aarch64_enc_ldarw(dst, mem));
7425
7426 ins_pipe(pipe_serial);
7427 %}
7428
7429 // Load Float
7430 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7431 %{
7432 match(Set dst (LoadF mem));
7433
7434 ins_cost(VOLATILE_REF_COST);
7435 format %{ "ldars $dst, $mem\t# float" %}
7436
7437 ins_encode( aarch64_enc_fldars(dst, mem) );
7438
7439 ins_pipe(pipe_serial);
7440 %}
7441
7442 // Load Double
7443 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7444 %{
7445 match(Set dst (LoadD mem));
7446
7447 ins_cost(VOLATILE_REF_COST);
7448 format %{ "ldard $dst, $mem\t# double" %}
7449
7450 ins_encode( aarch64_enc_fldard(dst, mem) );
7451
7452 ins_pipe(pipe_serial);
7453 %}
7454
7455 // Store Byte
7456 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7457 %{
7458 match(Set mem (StoreB mem src));
7459
7460 ins_cost(VOLATILE_REF_COST);
7461 format %{ "stlrb $src, $mem\t# byte" %}
7462
7463 ins_encode(aarch64_enc_stlrb(src, mem));
7464
7465 ins_pipe(pipe_class_memory);
7466 %}
7467
7468 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7469 %{
7470 match(Set mem (StoreB mem zero));
7471
7472 ins_cost(VOLATILE_REF_COST);
7473 format %{ "stlrb zr, $mem\t# byte" %}
7474
7475 ins_encode(aarch64_enc_stlrb0(mem));
7476
7477 ins_pipe(pipe_class_memory);
7478 %}
7479
7480 // Store Char/Short
7481 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7482 %{
7483 match(Set mem (StoreC mem src));
7484
7485 ins_cost(VOLATILE_REF_COST);
7486 format %{ "stlrh $src, $mem\t# short" %}
7487
7488 ins_encode(aarch64_enc_stlrh(src, mem));
7489
7490 ins_pipe(pipe_class_memory);
7491 %}
7492
7493 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7494 %{
7495 match(Set mem (StoreC mem zero));
7496
7497 ins_cost(VOLATILE_REF_COST);
7498 format %{ "stlrh zr, $mem\t# short" %}
7499
7500 ins_encode(aarch64_enc_stlrh0(mem));
7501
7502 ins_pipe(pipe_class_memory);
7503 %}
7504
7505 // Store Integer
7506
7507 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7508 %{
7509 match(Set mem(StoreI mem src));
7510
7511 ins_cost(VOLATILE_REF_COST);
7512 format %{ "stlrw $src, $mem\t# int" %}
7513
7514 ins_encode(aarch64_enc_stlrw(src, mem));
7515
7516 ins_pipe(pipe_class_memory);
7517 %}
7518
7519 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7520 %{
7521 match(Set mem(StoreI mem zero));
7522
7523 ins_cost(VOLATILE_REF_COST);
7524 format %{ "stlrw zr, $mem\t# int" %}
7525
7526 ins_encode(aarch64_enc_stlrw0(mem));
7527
7528 ins_pipe(pipe_class_memory);
7529 %}
7530
7531 // Store Long (64 bit signed)
7532 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7533 %{
7534 match(Set mem (StoreL mem src));
7535
7536 ins_cost(VOLATILE_REF_COST);
7537 format %{ "stlr $src, $mem\t# int" %}
7538
7539 ins_encode(aarch64_enc_stlr(src, mem));
7540
7541 ins_pipe(pipe_class_memory);
7542 %}
7543
7544 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7545 %{
7546 match(Set mem (StoreL mem zero));
7547
7548 ins_cost(VOLATILE_REF_COST);
7549 format %{ "stlr zr, $mem\t# int" %}
7550
7551 ins_encode(aarch64_enc_stlr0(mem));
7552
7553 ins_pipe(pipe_class_memory);
7554 %}
7555
7556 // Store Pointer
7557 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7558 %{
7559 match(Set mem (StoreP mem src));
7560 predicate(n->as_Store()->barrier_data() == 0);
7561
7562 ins_cost(VOLATILE_REF_COST);
7563 format %{ "stlr $src, $mem\t# ptr" %}
7564
7565 ins_encode(aarch64_enc_stlr(src, mem));
7566
7567 ins_pipe(pipe_class_memory);
7568 %}
7569
7570 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7571 %{
7572 match(Set mem (StoreP mem zero));
7573 predicate(n->as_Store()->barrier_data() == 0);
7574
7575 ins_cost(VOLATILE_REF_COST);
7576 format %{ "stlr zr, $mem\t# ptr" %}
7577
7578 ins_encode(aarch64_enc_stlr0(mem));
7579
7580 ins_pipe(pipe_class_memory);
7581 %}
7582
7583 // Store Compressed Pointer
7584 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7585 %{
7586 match(Set mem (StoreN mem src));
7587 predicate(n->as_Store()->barrier_data() == 0);
7588
7589 ins_cost(VOLATILE_REF_COST);
7590 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7591
7592 ins_encode(aarch64_enc_stlrw(src, mem));
7593
7594 ins_pipe(pipe_class_memory);
7595 %}
7596
7597 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7598 %{
7599 match(Set mem (StoreN mem zero));
7600 predicate(n->as_Store()->barrier_data() == 0);
7601
7602 ins_cost(VOLATILE_REF_COST);
7603 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7604
7605 ins_encode(aarch64_enc_stlrw0(mem));
7606
7607 ins_pipe(pipe_class_memory);
7608 %}
7609
7610 // Store Float
7611 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7612 %{
7613 match(Set mem (StoreF mem src));
7614
7615 ins_cost(VOLATILE_REF_COST);
7616 format %{ "stlrs $src, $mem\t# float" %}
7617
7618 ins_encode( aarch64_enc_fstlrs(src, mem) );
7619
7620 ins_pipe(pipe_class_memory);
7621 %}
7622
7623 // TODO
7624 // implement storeImmF0 and storeFImmPacked
7625
7626 // Store Double
7627 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7628 %{
7629 match(Set mem (StoreD mem src));
7630
7631 ins_cost(VOLATILE_REF_COST);
7632 format %{ "stlrd $src, $mem\t# double" %}
7633
7634 ins_encode( aarch64_enc_fstlrd(src, mem) );
7635
7636 ins_pipe(pipe_class_memory);
7637 %}
7638
7639 // ---------------- end of volatile loads and stores ----------------
7640
7641 instruct cacheWB(indirect addr)
7642 %{
7643 predicate(VM_Version::supports_data_cache_line_flush());
7644 match(CacheWB addr);
7645
7646 ins_cost(100);
7647 format %{"cache wb $addr" %}
7648 ins_encode %{
7649 assert($addr->index_position() < 0, "should be");
7650 assert($addr$$disp == 0, "should be");
7651 __ cache_wb(Address($addr$$base$$Register, 0));
7652 %}
7653 ins_pipe(pipe_slow); // XXX
7654 %}
7655
7656 instruct cacheWBPreSync()
7657 %{
7658 predicate(VM_Version::supports_data_cache_line_flush());
7659 match(CacheWBPreSync);
7660
7661 ins_cost(100);
7662 format %{"cache wb presync" %}
7663 ins_encode %{
7664 __ cache_wbsync(true);
7665 %}
7666 ins_pipe(pipe_slow); // XXX
7667 %}
7668
7669 instruct cacheWBPostSync()
7670 %{
7671 predicate(VM_Version::supports_data_cache_line_flush());
7672 match(CacheWBPostSync);
7673
7674 ins_cost(100);
7675 format %{"cache wb postsync" %}
7676 ins_encode %{
7677 __ cache_wbsync(false);
7678 %}
7679 ins_pipe(pipe_slow); // XXX
7680 %}
7681
7682 // ============================================================================
7683 // BSWAP Instructions
7684
7685 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7686 match(Set dst (ReverseBytesI src));
7687
7688 ins_cost(INSN_COST);
7689 format %{ "revw $dst, $src" %}
7690
7691 ins_encode %{
7692 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7693 %}
7694
7695 ins_pipe(ialu_reg);
7696 %}
7697
7698 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7699 match(Set dst (ReverseBytesL src));
7700
7701 ins_cost(INSN_COST);
7702 format %{ "rev $dst, $src" %}
7703
7704 ins_encode %{
7705 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7706 %}
7707
7708 ins_pipe(ialu_reg);
7709 %}
7710
7711 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7712 match(Set dst (ReverseBytesUS src));
7713
7714 ins_cost(INSN_COST);
7715 format %{ "rev16w $dst, $src" %}
7716
7717 ins_encode %{
7718 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7719 %}
7720
7721 ins_pipe(ialu_reg);
7722 %}
7723
7724 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7725 match(Set dst (ReverseBytesS src));
7726
7727 ins_cost(INSN_COST);
7728 format %{ "rev16w $dst, $src\n\t"
7729 "sbfmw $dst, $dst, #0, #15" %}
7730
7731 ins_encode %{
7732 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7733 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7734 %}
7735
7736 ins_pipe(ialu_reg);
7737 %}
7738
7739 // ============================================================================
7740 // Zero Count Instructions
7741
7742 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7743 match(Set dst (CountLeadingZerosI src));
7744
7745 ins_cost(INSN_COST);
7746 format %{ "clzw $dst, $src" %}
7747 ins_encode %{
7748 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7749 %}
7750
7751 ins_pipe(ialu_reg);
7752 %}
7753
7754 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7755 match(Set dst (CountLeadingZerosL src));
7756
7757 ins_cost(INSN_COST);
7758 format %{ "clz $dst, $src" %}
7759 ins_encode %{
7760 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7761 %}
7762
7763 ins_pipe(ialu_reg);
7764 %}
7765
7766 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7767 match(Set dst (CountTrailingZerosI src));
7768
7769 ins_cost(INSN_COST * 2);
7770 format %{ "rbitw $dst, $src\n\t"
7771 "clzw $dst, $dst" %}
7772 ins_encode %{
7773 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7774 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7775 %}
7776
7777 ins_pipe(ialu_reg);
7778 %}
7779
7780 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7781 match(Set dst (CountTrailingZerosL src));
7782
7783 ins_cost(INSN_COST * 2);
7784 format %{ "rbit $dst, $src\n\t"
7785 "clz $dst, $dst" %}
7786 ins_encode %{
7787 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7788 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7789 %}
7790
7791 ins_pipe(ialu_reg);
7792 %}
7793
7794 //---------- Population Count Instructions -------------------------------------
7795 //
7796
7797 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7798 match(Set dst (PopCountI src));
7799 effect(TEMP tmp);
7800 ins_cost(INSN_COST * 13);
7801
7802 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7803 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7804 "addv $tmp, $tmp\t# vector (8B)\n\t"
7805 "mov $dst, $tmp\t# vector (1D)" %}
7806 ins_encode %{
7807 __ fmovs($tmp$$FloatRegister, $src$$Register);
7808 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7809 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7810 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7811 %}
7812
7813 ins_pipe(pipe_class_default);
7814 %}
7815
7816 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7817 match(Set dst (PopCountI (LoadI mem)));
7818 effect(TEMP tmp);
7819 ins_cost(INSN_COST * 13);
7820
7821 format %{ "ldrs $tmp, $mem\n\t"
7822 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7823 "addv $tmp, $tmp\t# vector (8B)\n\t"
7824 "mov $dst, $tmp\t# vector (1D)" %}
7825 ins_encode %{
7826 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7827 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7828 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7829 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7830 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7831 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7832 %}
7833
7834 ins_pipe(pipe_class_default);
7835 %}
7836
7837 // Note: Long.bitCount(long) returns an int.
7838 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7839 match(Set dst (PopCountL src));
7840 effect(TEMP tmp);
7841 ins_cost(INSN_COST * 13);
7842
7843 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7844 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7845 "addv $tmp, $tmp\t# vector (8B)\n\t"
7846 "mov $dst, $tmp\t# vector (1D)" %}
7847 ins_encode %{
7848 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7849 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7850 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7851 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7852 %}
7853
7854 ins_pipe(pipe_class_default);
7855 %}
7856
7857 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7858 match(Set dst (PopCountL (LoadL mem)));
7859 effect(TEMP tmp);
7860 ins_cost(INSN_COST * 13);
7861
7862 format %{ "ldrd $tmp, $mem\n\t"
7863 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7864 "addv $tmp, $tmp\t# vector (8B)\n\t"
7865 "mov $dst, $tmp\t# vector (1D)" %}
7866 ins_encode %{
7867 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7868 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7869 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7870 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7871 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7872 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7873 %}
7874
7875 ins_pipe(pipe_class_default);
7876 %}
7877
7878 // ============================================================================
7879 // VerifyVectorAlignment Instruction
7880
7881 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7882 match(Set addr (VerifyVectorAlignment addr mask));
7883 effect(KILL cr);
7884 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7885 ins_encode %{
7886 Label Lskip;
7887 // check if masked bits of addr are zero
7888 __ tst($addr$$Register, $mask$$constant);
7889 __ br(Assembler::EQ, Lskip);
7890 __ stop("verify_vector_alignment found a misaligned vector memory access");
7891 __ bind(Lskip);
7892 %}
7893 ins_pipe(pipe_slow);
7894 %}
7895
7896 // ============================================================================
7897 // MemBar Instruction
7898
7899 instruct load_fence() %{
7900 match(LoadFence);
7901 ins_cost(VOLATILE_REF_COST);
7902
7903 format %{ "load_fence" %}
7904
7905 ins_encode %{
7906 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7907 %}
7908 ins_pipe(pipe_serial);
7909 %}
7910
7911 instruct unnecessary_membar_acquire() %{
7912 predicate(unnecessary_acquire(n));
7913 match(MemBarAcquire);
7914 ins_cost(0);
7915
7916 format %{ "membar_acquire (elided)" %}
7917
7918 ins_encode %{
7919 __ block_comment("membar_acquire (elided)");
7920 %}
7921
7922 ins_pipe(pipe_class_empty);
7923 %}
7924
7925 instruct membar_acquire() %{
7926 match(MemBarAcquire);
7927 ins_cost(VOLATILE_REF_COST);
7928
7929 format %{ "membar_acquire\n\t"
7930 "dmb ishld" %}
7931
7932 ins_encode %{
7933 __ block_comment("membar_acquire");
7934 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7935 %}
7936
7937 ins_pipe(pipe_serial);
7938 %}
7939
7940
7941 instruct membar_acquire_lock() %{
7942 match(MemBarAcquireLock);
7943 ins_cost(VOLATILE_REF_COST);
7944
7945 format %{ "membar_acquire_lock (elided)" %}
7946
7947 ins_encode %{
7948 __ block_comment("membar_acquire_lock (elided)");
7949 %}
7950
7951 ins_pipe(pipe_serial);
7952 %}
7953
7954 instruct store_fence() %{
7955 match(StoreFence);
7956 ins_cost(VOLATILE_REF_COST);
7957
7958 format %{ "store_fence" %}
7959
7960 ins_encode %{
7961 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7962 %}
7963 ins_pipe(pipe_serial);
7964 %}
7965
7966 instruct unnecessary_membar_release() %{
7967 predicate(unnecessary_release(n));
7968 match(MemBarRelease);
7969 ins_cost(0);
7970
7971 format %{ "membar_release (elided)" %}
7972
7973 ins_encode %{
7974 __ block_comment("membar_release (elided)");
7975 %}
7976 ins_pipe(pipe_serial);
7977 %}
7978
7979 instruct membar_release() %{
7980 match(MemBarRelease);
7981 ins_cost(VOLATILE_REF_COST);
7982
7983 format %{ "membar_release\n\t"
7984 "dmb ishst\n\tdmb ishld" %}
7985
7986 ins_encode %{
7987 __ block_comment("membar_release");
7988 // These will be merged if AlwaysMergeDMB is enabled.
7989 __ membar(Assembler::StoreStore);
7990 __ membar(Assembler::LoadStore);
7991 %}
7992 ins_pipe(pipe_serial);
7993 %}
7994
7995 instruct membar_storestore() %{
7996 match(MemBarStoreStore);
7997 match(StoreStoreFence);
7998 ins_cost(VOLATILE_REF_COST);
7999
8000 format %{ "MEMBAR-store-store" %}
8001
8002 ins_encode %{
8003 __ membar(Assembler::StoreStore);
8004 %}
8005 ins_pipe(pipe_serial);
8006 %}
8007
8008 instruct membar_release_lock() %{
8009 match(MemBarReleaseLock);
8010 ins_cost(VOLATILE_REF_COST);
8011
8012 format %{ "membar_release_lock (elided)" %}
8013
8014 ins_encode %{
8015 __ block_comment("membar_release_lock (elided)");
8016 %}
8017
8018 ins_pipe(pipe_serial);
8019 %}
8020
8021 instruct unnecessary_membar_volatile() %{
8022 predicate(unnecessary_volatile(n));
8023 match(MemBarVolatile);
8024 ins_cost(0);
8025
8026 format %{ "membar_volatile (elided)" %}
8027
8028 ins_encode %{
8029 __ block_comment("membar_volatile (elided)");
8030 %}
8031
8032 ins_pipe(pipe_serial);
8033 %}
8034
8035 instruct membar_volatile() %{
8036 match(MemBarVolatile);
8037 ins_cost(VOLATILE_REF_COST*100);
8038
8039 format %{ "membar_volatile\n\t"
8040 "dmb ish"%}
8041
8042 ins_encode %{
8043 __ block_comment("membar_volatile");
8044 __ membar(Assembler::StoreLoad);
8045 %}
8046
8047 ins_pipe(pipe_serial);
8048 %}
8049
8050 // ============================================================================
8051 // Cast/Convert Instructions
8052
8053 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8054 match(Set dst (CastX2P src));
8055
8056 ins_cost(INSN_COST);
8057 format %{ "mov $dst, $src\t# long -> ptr" %}
8058
8059 ins_encode %{
8060 if ($dst$$reg != $src$$reg) {
8061 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8062 }
8063 %}
8064
8065 ins_pipe(ialu_reg);
8066 %}
8067
8068 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8069 match(Set dst (CastP2X src));
8070
8071 ins_cost(INSN_COST);
8072 format %{ "mov $dst, $src\t# ptr -> long" %}
8073
8074 ins_encode %{
8075 if ($dst$$reg != $src$$reg) {
8076 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8077 }
8078 %}
8079
8080 ins_pipe(ialu_reg);
8081 %}
8082
8083 // Convert oop into int for vectors alignment masking
8084 instruct convP2I(iRegINoSp dst, iRegP src) %{
8085 match(Set dst (ConvL2I (CastP2X src)));
8086
8087 ins_cost(INSN_COST);
8088 format %{ "movw $dst, $src\t# ptr -> int" %}
8089 ins_encode %{
8090 __ movw($dst$$Register, $src$$Register);
8091 %}
8092
8093 ins_pipe(ialu_reg);
8094 %}
8095
8096 // Convert compressed oop into int for vectors alignment masking
8097 // in case of 32bit oops (heap < 4Gb).
8098 instruct convN2I(iRegINoSp dst, iRegN src)
8099 %{
8100 predicate(CompressedOops::shift() == 0);
8101 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8102
8103 ins_cost(INSN_COST);
8104 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8105 ins_encode %{
8106 __ movw($dst$$Register, $src$$Register);
8107 %}
8108
8109 ins_pipe(ialu_reg);
8110 %}
8111
8112
8113 // Convert oop pointer into compressed form
8114 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8115 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8116 match(Set dst (EncodeP src));
8117 effect(KILL cr);
8118 ins_cost(INSN_COST * 3);
8119 format %{ "encode_heap_oop $dst, $src" %}
8120 ins_encode %{
8121 Register s = $src$$Register;
8122 Register d = $dst$$Register;
8123 __ encode_heap_oop(d, s);
8124 %}
8125 ins_pipe(ialu_reg);
8126 %}
8127
8128 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8129 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8130 match(Set dst (EncodeP src));
8131 ins_cost(INSN_COST * 3);
8132 format %{ "encode_heap_oop_not_null $dst, $src" %}
8133 ins_encode %{
8134 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8135 %}
8136 ins_pipe(ialu_reg);
8137 %}
8138
8139 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8140 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8141 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8142 match(Set dst (DecodeN src));
8143 ins_cost(INSN_COST * 3);
8144 format %{ "decode_heap_oop $dst, $src" %}
8145 ins_encode %{
8146 Register s = $src$$Register;
8147 Register d = $dst$$Register;
8148 __ decode_heap_oop(d, s);
8149 %}
8150 ins_pipe(ialu_reg);
8151 %}
8152
8153 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8154 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8155 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8156 match(Set dst (DecodeN src));
8157 ins_cost(INSN_COST * 3);
8158 format %{ "decode_heap_oop_not_null $dst, $src" %}
8159 ins_encode %{
8160 Register s = $src$$Register;
8161 Register d = $dst$$Register;
8162 __ decode_heap_oop_not_null(d, s);
8163 %}
8164 ins_pipe(ialu_reg);
8165 %}
8166
8167 // n.b. AArch64 implementations of encode_klass_not_null and
8168 // decode_klass_not_null do not modify the flags register so, unlike
8169 // Intel, we don't kill CR as a side effect here
8170
8171 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8172 match(Set dst (EncodePKlass src));
8173
8174 ins_cost(INSN_COST * 3);
8175 format %{ "encode_klass_not_null $dst,$src" %}
8176
8177 ins_encode %{
8178 Register src_reg = as_Register($src$$reg);
8179 Register dst_reg = as_Register($dst$$reg);
8180 __ encode_klass_not_null(dst_reg, src_reg);
8181 %}
8182
8183 ins_pipe(ialu_reg);
8184 %}
8185
8186 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8187 match(Set dst (DecodeNKlass src));
8188
8189 ins_cost(INSN_COST * 3);
8190 format %{ "decode_klass_not_null $dst,$src" %}
8191
8192 ins_encode %{
8193 Register src_reg = as_Register($src$$reg);
8194 Register dst_reg = as_Register($dst$$reg);
8195 if (dst_reg != src_reg) {
8196 __ decode_klass_not_null(dst_reg, src_reg);
8197 } else {
8198 __ decode_klass_not_null(dst_reg);
8199 }
8200 %}
8201
8202 ins_pipe(ialu_reg);
8203 %}
8204
8205 instruct checkCastPP(iRegPNoSp dst)
8206 %{
8207 match(Set dst (CheckCastPP dst));
8208
8209 size(0);
8210 format %{ "# checkcastPP of $dst" %}
8211 ins_encode(/* empty encoding */);
8212 ins_pipe(pipe_class_empty);
8213 %}
8214
8215 instruct castPP(iRegPNoSp dst)
8216 %{
8217 match(Set dst (CastPP dst));
8218
8219 size(0);
8220 format %{ "# castPP of $dst" %}
8221 ins_encode(/* empty encoding */);
8222 ins_pipe(pipe_class_empty);
8223 %}
8224
8225 instruct castII(iRegI dst)
8226 %{
8227 predicate(VerifyConstraintCasts == 0);
8228 match(Set dst (CastII dst));
8229
8230 size(0);
8231 format %{ "# castII of $dst" %}
8232 ins_encode(/* empty encoding */);
8233 ins_cost(0);
8234 ins_pipe(pipe_class_empty);
8235 %}
8236
8237 instruct castII_checked(iRegI dst, rFlagsReg cr)
8238 %{
8239 predicate(VerifyConstraintCasts > 0);
8240 match(Set dst (CastII dst));
8241 effect(KILL cr);
8242
8243 format %{ "# castII_checked of $dst" %}
8244 ins_encode %{
8245 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8246 %}
8247 ins_pipe(pipe_slow);
8248 %}
8249
8250 instruct castLL(iRegL dst)
8251 %{
8252 predicate(VerifyConstraintCasts == 0);
8253 match(Set dst (CastLL dst));
8254
8255 size(0);
8256 format %{ "# castLL of $dst" %}
8257 ins_encode(/* empty encoding */);
8258 ins_cost(0);
8259 ins_pipe(pipe_class_empty);
8260 %}
8261
8262 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8263 %{
8264 predicate(VerifyConstraintCasts > 0);
8265 match(Set dst (CastLL dst));
8266 effect(KILL cr);
8267
8268 format %{ "# castLL_checked of $dst" %}
8269 ins_encode %{
8270 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8271 %}
8272 ins_pipe(pipe_slow);
8273 %}
8274
8275 instruct castHH(vRegF dst)
8276 %{
8277 match(Set dst (CastHH dst));
8278 size(0);
8279 format %{ "# castHH of $dst" %}
8280 ins_encode(/* empty encoding */);
8281 ins_cost(0);
8282 ins_pipe(pipe_class_empty);
8283 %}
8284
8285 instruct castFF(vRegF dst)
8286 %{
8287 match(Set dst (CastFF dst));
8288
8289 size(0);
8290 format %{ "# castFF of $dst" %}
8291 ins_encode(/* empty encoding */);
8292 ins_cost(0);
8293 ins_pipe(pipe_class_empty);
8294 %}
8295
8296 instruct castDD(vRegD dst)
8297 %{
8298 match(Set dst (CastDD dst));
8299
8300 size(0);
8301 format %{ "# castDD of $dst" %}
8302 ins_encode(/* empty encoding */);
8303 ins_cost(0);
8304 ins_pipe(pipe_class_empty);
8305 %}
8306
8307 instruct castVV(vReg dst)
8308 %{
8309 match(Set dst (CastVV dst));
8310
8311 size(0);
8312 format %{ "# castVV of $dst" %}
8313 ins_encode(/* empty encoding */);
8314 ins_cost(0);
8315 ins_pipe(pipe_class_empty);
8316 %}
8317
8318 instruct castVVMask(pRegGov dst)
8319 %{
8320 match(Set dst (CastVV dst));
8321
8322 size(0);
8323 format %{ "# castVV of $dst" %}
8324 ins_encode(/* empty encoding */);
8325 ins_cost(0);
8326 ins_pipe(pipe_class_empty);
8327 %}
8328
8329 // Manifest a CmpU result in an integer register.
8330 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8331 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8332 %{
8333 match(Set dst (CmpU3 src1 src2));
8334 effect(KILL flags);
8335
8336 ins_cost(INSN_COST * 3);
8337 format %{
8338 "cmpw $src1, $src2\n\t"
8339 "csetw $dst, ne\n\t"
8340 "cnegw $dst, lo\t# CmpU3(reg)"
8341 %}
8342 ins_encode %{
8343 __ cmpw($src1$$Register, $src2$$Register);
8344 __ csetw($dst$$Register, Assembler::NE);
8345 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8346 %}
8347
8348 ins_pipe(pipe_class_default);
8349 %}
8350
8351 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8352 %{
8353 match(Set dst (CmpU3 src1 src2));
8354 effect(KILL flags);
8355
8356 ins_cost(INSN_COST * 3);
8357 format %{
8358 "subsw zr, $src1, $src2\n\t"
8359 "csetw $dst, ne\n\t"
8360 "cnegw $dst, lo\t# CmpU3(imm)"
8361 %}
8362 ins_encode %{
8363 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8364 __ csetw($dst$$Register, Assembler::NE);
8365 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8366 %}
8367
8368 ins_pipe(pipe_class_default);
8369 %}
8370
8371 // Manifest a CmpUL result in an integer register.
8372 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8373 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8374 %{
8375 match(Set dst (CmpUL3 src1 src2));
8376 effect(KILL flags);
8377
8378 ins_cost(INSN_COST * 3);
8379 format %{
8380 "cmp $src1, $src2\n\t"
8381 "csetw $dst, ne\n\t"
8382 "cnegw $dst, lo\t# CmpUL3(reg)"
8383 %}
8384 ins_encode %{
8385 __ cmp($src1$$Register, $src2$$Register);
8386 __ csetw($dst$$Register, Assembler::NE);
8387 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8388 %}
8389
8390 ins_pipe(pipe_class_default);
8391 %}
8392
8393 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8394 %{
8395 match(Set dst (CmpUL3 src1 src2));
8396 effect(KILL flags);
8397
8398 ins_cost(INSN_COST * 3);
8399 format %{
8400 "subs zr, $src1, $src2\n\t"
8401 "csetw $dst, ne\n\t"
8402 "cnegw $dst, lo\t# CmpUL3(imm)"
8403 %}
8404 ins_encode %{
8405 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8406 __ csetw($dst$$Register, Assembler::NE);
8407 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8408 %}
8409
8410 ins_pipe(pipe_class_default);
8411 %}
8412
8413 // Manifest a CmpL result in an integer register.
8414 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8415 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8416 %{
8417 match(Set dst (CmpL3 src1 src2));
8418 effect(KILL flags);
8419
8420 ins_cost(INSN_COST * 3);
8421 format %{
8422 "cmp $src1, $src2\n\t"
8423 "csetw $dst, ne\n\t"
8424 "cnegw $dst, lt\t# CmpL3(reg)"
8425 %}
8426 ins_encode %{
8427 __ cmp($src1$$Register, $src2$$Register);
8428 __ csetw($dst$$Register, Assembler::NE);
8429 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8430 %}
8431
8432 ins_pipe(pipe_class_default);
8433 %}
8434
8435 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8436 %{
8437 match(Set dst (CmpL3 src1 src2));
8438 effect(KILL flags);
8439
8440 ins_cost(INSN_COST * 3);
8441 format %{
8442 "subs zr, $src1, $src2\n\t"
8443 "csetw $dst, ne\n\t"
8444 "cnegw $dst, lt\t# CmpL3(imm)"
8445 %}
8446 ins_encode %{
8447 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8448 __ csetw($dst$$Register, Assembler::NE);
8449 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8450 %}
8451
8452 ins_pipe(pipe_class_default);
8453 %}
8454
8455 // ============================================================================
8456 // Conditional Move Instructions
8457
8458 // n.b. we have identical rules for both a signed compare op (cmpOp)
8459 // and an unsigned compare op (cmpOpU). it would be nice if we could
8460 // define an op class which merged both inputs and use it to type the
8461 // argument to a single rule. unfortunatelyt his fails because the
8462 // opclass does not live up to the COND_INTER interface of its
8463 // component operands. When the generic code tries to negate the
8464 // operand it ends up running the generci Machoper::negate method
8465 // which throws a ShouldNotHappen. So, we have to provide two flavours
8466 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8467
8468 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8469 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8470
8471 ins_cost(INSN_COST * 2);
8472 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8473
8474 ins_encode %{
8475 __ cselw(as_Register($dst$$reg),
8476 as_Register($src2$$reg),
8477 as_Register($src1$$reg),
8478 (Assembler::Condition)$cmp$$cmpcode);
8479 %}
8480
8481 ins_pipe(icond_reg_reg);
8482 %}
8483
8484 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8485 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8486
8487 ins_cost(INSN_COST * 2);
8488 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8489
8490 ins_encode %{
8491 __ cselw(as_Register($dst$$reg),
8492 as_Register($src2$$reg),
8493 as_Register($src1$$reg),
8494 (Assembler::Condition)$cmp$$cmpcode);
8495 %}
8496
8497 ins_pipe(icond_reg_reg);
8498 %}
8499
8500 // special cases where one arg is zero
8501
8502 // n.b. this is selected in preference to the rule above because it
8503 // avoids loading constant 0 into a source register
8504
8505 // TODO
8506 // we ought only to be able to cull one of these variants as the ideal
8507 // transforms ought always to order the zero consistently (to left/right?)
8508
8509 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8510 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8511
8512 ins_cost(INSN_COST * 2);
8513 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8514
8515 ins_encode %{
8516 __ cselw(as_Register($dst$$reg),
8517 as_Register($src$$reg),
8518 zr,
8519 (Assembler::Condition)$cmp$$cmpcode);
8520 %}
8521
8522 ins_pipe(icond_reg);
8523 %}
8524
8525 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8526 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8527
8528 ins_cost(INSN_COST * 2);
8529 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8530
8531 ins_encode %{
8532 __ cselw(as_Register($dst$$reg),
8533 as_Register($src$$reg),
8534 zr,
8535 (Assembler::Condition)$cmp$$cmpcode);
8536 %}
8537
8538 ins_pipe(icond_reg);
8539 %}
8540
8541 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8542 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8543
8544 ins_cost(INSN_COST * 2);
8545 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8546
8547 ins_encode %{
8548 __ cselw(as_Register($dst$$reg),
8549 zr,
8550 as_Register($src$$reg),
8551 (Assembler::Condition)$cmp$$cmpcode);
8552 %}
8553
8554 ins_pipe(icond_reg);
8555 %}
8556
8557 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8558 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8559
8560 ins_cost(INSN_COST * 2);
8561 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8562
8563 ins_encode %{
8564 __ cselw(as_Register($dst$$reg),
8565 zr,
8566 as_Register($src$$reg),
8567 (Assembler::Condition)$cmp$$cmpcode);
8568 %}
8569
8570 ins_pipe(icond_reg);
8571 %}
8572
8573 // special case for creating a boolean 0 or 1
8574
8575 // n.b. this is selected in preference to the rule above because it
8576 // avoids loading constants 0 and 1 into a source register
8577
8578 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8579 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8580
8581 ins_cost(INSN_COST * 2);
8582 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8583
8584 ins_encode %{
8585 // equivalently
8586 // cset(as_Register($dst$$reg),
8587 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8588 __ csincw(as_Register($dst$$reg),
8589 zr,
8590 zr,
8591 (Assembler::Condition)$cmp$$cmpcode);
8592 %}
8593
8594 ins_pipe(icond_none);
8595 %}
8596
8597 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8598 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8599
8600 ins_cost(INSN_COST * 2);
8601 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8602
8603 ins_encode %{
8604 // equivalently
8605 // cset(as_Register($dst$$reg),
8606 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8607 __ csincw(as_Register($dst$$reg),
8608 zr,
8609 zr,
8610 (Assembler::Condition)$cmp$$cmpcode);
8611 %}
8612
8613 ins_pipe(icond_none);
8614 %}
8615
8616 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8617 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8618
8619 ins_cost(INSN_COST * 2);
8620 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8621
8622 ins_encode %{
8623 __ csel(as_Register($dst$$reg),
8624 as_Register($src2$$reg),
8625 as_Register($src1$$reg),
8626 (Assembler::Condition)$cmp$$cmpcode);
8627 %}
8628
8629 ins_pipe(icond_reg_reg);
8630 %}
8631
8632 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8633 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8634
8635 ins_cost(INSN_COST * 2);
8636 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8637
8638 ins_encode %{
8639 __ csel(as_Register($dst$$reg),
8640 as_Register($src2$$reg),
8641 as_Register($src1$$reg),
8642 (Assembler::Condition)$cmp$$cmpcode);
8643 %}
8644
8645 ins_pipe(icond_reg_reg);
8646 %}
8647
8648 // special cases where one arg is zero
8649
8650 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8651 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8652
8653 ins_cost(INSN_COST * 2);
8654 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8655
8656 ins_encode %{
8657 __ csel(as_Register($dst$$reg),
8658 zr,
8659 as_Register($src$$reg),
8660 (Assembler::Condition)$cmp$$cmpcode);
8661 %}
8662
8663 ins_pipe(icond_reg);
8664 %}
8665
8666 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8667 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8668
8669 ins_cost(INSN_COST * 2);
8670 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8671
8672 ins_encode %{
8673 __ csel(as_Register($dst$$reg),
8674 zr,
8675 as_Register($src$$reg),
8676 (Assembler::Condition)$cmp$$cmpcode);
8677 %}
8678
8679 ins_pipe(icond_reg);
8680 %}
8681
8682 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8683 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8684
8685 ins_cost(INSN_COST * 2);
8686 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8687
8688 ins_encode %{
8689 __ csel(as_Register($dst$$reg),
8690 as_Register($src$$reg),
8691 zr,
8692 (Assembler::Condition)$cmp$$cmpcode);
8693 %}
8694
8695 ins_pipe(icond_reg);
8696 %}
8697
8698 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8699 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8700
8701 ins_cost(INSN_COST * 2);
8702 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8703
8704 ins_encode %{
8705 __ csel(as_Register($dst$$reg),
8706 as_Register($src$$reg),
8707 zr,
8708 (Assembler::Condition)$cmp$$cmpcode);
8709 %}
8710
8711 ins_pipe(icond_reg);
8712 %}
8713
8714 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8715 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8716
8717 ins_cost(INSN_COST * 2);
8718 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8719
8720 ins_encode %{
8721 __ csel(as_Register($dst$$reg),
8722 as_Register($src2$$reg),
8723 as_Register($src1$$reg),
8724 (Assembler::Condition)$cmp$$cmpcode);
8725 %}
8726
8727 ins_pipe(icond_reg_reg);
8728 %}
8729
8730 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8731 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8732
8733 ins_cost(INSN_COST * 2);
8734 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8735
8736 ins_encode %{
8737 __ csel(as_Register($dst$$reg),
8738 as_Register($src2$$reg),
8739 as_Register($src1$$reg),
8740 (Assembler::Condition)$cmp$$cmpcode);
8741 %}
8742
8743 ins_pipe(icond_reg_reg);
8744 %}
8745
8746 // special cases where one arg is zero
8747
8748 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8749 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8750
8751 ins_cost(INSN_COST * 2);
8752 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8753
8754 ins_encode %{
8755 __ csel(as_Register($dst$$reg),
8756 zr,
8757 as_Register($src$$reg),
8758 (Assembler::Condition)$cmp$$cmpcode);
8759 %}
8760
8761 ins_pipe(icond_reg);
8762 %}
8763
8764 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8765 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8766
8767 ins_cost(INSN_COST * 2);
8768 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8769
8770 ins_encode %{
8771 __ csel(as_Register($dst$$reg),
8772 zr,
8773 as_Register($src$$reg),
8774 (Assembler::Condition)$cmp$$cmpcode);
8775 %}
8776
8777 ins_pipe(icond_reg);
8778 %}
8779
8780 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8781 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8782
8783 ins_cost(INSN_COST * 2);
8784 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8785
8786 ins_encode %{
8787 __ csel(as_Register($dst$$reg),
8788 as_Register($src$$reg),
8789 zr,
8790 (Assembler::Condition)$cmp$$cmpcode);
8791 %}
8792
8793 ins_pipe(icond_reg);
8794 %}
8795
8796 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8797 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8798
8799 ins_cost(INSN_COST * 2);
8800 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8801
8802 ins_encode %{
8803 __ csel(as_Register($dst$$reg),
8804 as_Register($src$$reg),
8805 zr,
8806 (Assembler::Condition)$cmp$$cmpcode);
8807 %}
8808
8809 ins_pipe(icond_reg);
8810 %}
8811
8812 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8813 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8814
8815 ins_cost(INSN_COST * 2);
8816 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8817
8818 ins_encode %{
8819 __ cselw(as_Register($dst$$reg),
8820 as_Register($src2$$reg),
8821 as_Register($src1$$reg),
8822 (Assembler::Condition)$cmp$$cmpcode);
8823 %}
8824
8825 ins_pipe(icond_reg_reg);
8826 %}
8827
8828 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8829 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8830
8831 ins_cost(INSN_COST * 2);
8832 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8833
8834 ins_encode %{
8835 __ cselw(as_Register($dst$$reg),
8836 as_Register($src2$$reg),
8837 as_Register($src1$$reg),
8838 (Assembler::Condition)$cmp$$cmpcode);
8839 %}
8840
8841 ins_pipe(icond_reg_reg);
8842 %}
8843
8844 // special cases where one arg is zero
8845
8846 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8847 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8848
8849 ins_cost(INSN_COST * 2);
8850 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8851
8852 ins_encode %{
8853 __ cselw(as_Register($dst$$reg),
8854 zr,
8855 as_Register($src$$reg),
8856 (Assembler::Condition)$cmp$$cmpcode);
8857 %}
8858
8859 ins_pipe(icond_reg);
8860 %}
8861
8862 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8863 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8864
8865 ins_cost(INSN_COST * 2);
8866 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8867
8868 ins_encode %{
8869 __ cselw(as_Register($dst$$reg),
8870 zr,
8871 as_Register($src$$reg),
8872 (Assembler::Condition)$cmp$$cmpcode);
8873 %}
8874
8875 ins_pipe(icond_reg);
8876 %}
8877
8878 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8879 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8880
8881 ins_cost(INSN_COST * 2);
8882 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8883
8884 ins_encode %{
8885 __ cselw(as_Register($dst$$reg),
8886 as_Register($src$$reg),
8887 zr,
8888 (Assembler::Condition)$cmp$$cmpcode);
8889 %}
8890
8891 ins_pipe(icond_reg);
8892 %}
8893
8894 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8895 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8896
8897 ins_cost(INSN_COST * 2);
8898 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8899
8900 ins_encode %{
8901 __ cselw(as_Register($dst$$reg),
8902 as_Register($src$$reg),
8903 zr,
8904 (Assembler::Condition)$cmp$$cmpcode);
8905 %}
8906
8907 ins_pipe(icond_reg);
8908 %}
8909
8910 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8911 %{
8912 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8913
8914 ins_cost(INSN_COST * 3);
8915
8916 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8917 ins_encode %{
8918 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8919 __ fcsels(as_FloatRegister($dst$$reg),
8920 as_FloatRegister($src2$$reg),
8921 as_FloatRegister($src1$$reg),
8922 cond);
8923 %}
8924
8925 ins_pipe(fp_cond_reg_reg_s);
8926 %}
8927
8928 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8929 %{
8930 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8931
8932 ins_cost(INSN_COST * 3);
8933
8934 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8935 ins_encode %{
8936 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8937 __ fcsels(as_FloatRegister($dst$$reg),
8938 as_FloatRegister($src2$$reg),
8939 as_FloatRegister($src1$$reg),
8940 cond);
8941 %}
8942
8943 ins_pipe(fp_cond_reg_reg_s);
8944 %}
8945
8946 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8947 %{
8948 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8949
8950 ins_cost(INSN_COST * 3);
8951
8952 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8953 ins_encode %{
8954 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8955 __ fcseld(as_FloatRegister($dst$$reg),
8956 as_FloatRegister($src2$$reg),
8957 as_FloatRegister($src1$$reg),
8958 cond);
8959 %}
8960
8961 ins_pipe(fp_cond_reg_reg_d);
8962 %}
8963
8964 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
8965 %{
8966 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8967
8968 ins_cost(INSN_COST * 3);
8969
8970 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8971 ins_encode %{
8972 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8973 __ fcseld(as_FloatRegister($dst$$reg),
8974 as_FloatRegister($src2$$reg),
8975 as_FloatRegister($src1$$reg),
8976 cond);
8977 %}
8978
8979 ins_pipe(fp_cond_reg_reg_d);
8980 %}
8981
8982 // ============================================================================
8983 // Arithmetic Instructions
8984 //
8985
8986 // Integer Addition
8987
8988 // TODO
8989 // these currently employ operations which do not set CR and hence are
8990 // not flagged as killing CR but we would like to isolate the cases
8991 // where we want to set flags from those where we don't. need to work
8992 // out how to do that.
8993
8994 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8995 match(Set dst (AddI src1 src2));
8996
8997 ins_cost(INSN_COST);
8998 format %{ "addw $dst, $src1, $src2" %}
8999
9000 ins_encode %{
9001 __ addw(as_Register($dst$$reg),
9002 as_Register($src1$$reg),
9003 as_Register($src2$$reg));
9004 %}
9005
9006 ins_pipe(ialu_reg_reg);
9007 %}
9008
9009 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9010 match(Set dst (AddI src1 src2));
9011
9012 ins_cost(INSN_COST);
9013 format %{ "addw $dst, $src1, $src2" %}
9014
9015 // use opcode to indicate that this is an add not a sub
9016 opcode(0x0);
9017
9018 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9019
9020 ins_pipe(ialu_reg_imm);
9021 %}
9022
9023 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9024 match(Set dst (AddI (ConvL2I src1) src2));
9025
9026 ins_cost(INSN_COST);
9027 format %{ "addw $dst, $src1, $src2" %}
9028
9029 // use opcode to indicate that this is an add not a sub
9030 opcode(0x0);
9031
9032 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9033
9034 ins_pipe(ialu_reg_imm);
9035 %}
9036
9037 // Pointer Addition
9038 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9039 match(Set dst (AddP src1 src2));
9040
9041 ins_cost(INSN_COST);
9042 format %{ "add $dst, $src1, $src2\t# ptr" %}
9043
9044 ins_encode %{
9045 __ add(as_Register($dst$$reg),
9046 as_Register($src1$$reg),
9047 as_Register($src2$$reg));
9048 %}
9049
9050 ins_pipe(ialu_reg_reg);
9051 %}
9052
9053 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9054 match(Set dst (AddP src1 (ConvI2L src2)));
9055
9056 ins_cost(1.9 * INSN_COST);
9057 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9058
9059 ins_encode %{
9060 __ add(as_Register($dst$$reg),
9061 as_Register($src1$$reg),
9062 as_Register($src2$$reg), ext::sxtw);
9063 %}
9064
9065 ins_pipe(ialu_reg_reg);
9066 %}
9067
9068 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9069 match(Set dst (AddP src1 (LShiftL src2 scale)));
9070
9071 ins_cost(1.9 * INSN_COST);
9072 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9073
9074 ins_encode %{
9075 __ lea(as_Register($dst$$reg),
9076 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9077 Address::lsl($scale$$constant)));
9078 %}
9079
9080 ins_pipe(ialu_reg_reg_shift);
9081 %}
9082
9083 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9084 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9085
9086 ins_cost(1.9 * INSN_COST);
9087 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9088
9089 ins_encode %{
9090 __ lea(as_Register($dst$$reg),
9091 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9092 Address::sxtw($scale$$constant)));
9093 %}
9094
9095 ins_pipe(ialu_reg_reg_shift);
9096 %}
9097
9098 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9099 match(Set dst (LShiftL (ConvI2L src) scale));
9100
9101 ins_cost(INSN_COST);
9102 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9103
9104 ins_encode %{
9105 __ sbfiz(as_Register($dst$$reg),
9106 as_Register($src$$reg),
9107 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9108 %}
9109
9110 ins_pipe(ialu_reg_shift);
9111 %}
9112
9113 // Pointer Immediate Addition
9114 // n.b. this needs to be more expensive than using an indirect memory
9115 // operand
9116 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9117 match(Set dst (AddP src1 src2));
9118
9119 ins_cost(INSN_COST);
9120 format %{ "add $dst, $src1, $src2\t# ptr" %}
9121
9122 // use opcode to indicate that this is an add not a sub
9123 opcode(0x0);
9124
9125 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9126
9127 ins_pipe(ialu_reg_imm);
9128 %}
9129
9130 // Long Addition
9131 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9132
9133 match(Set dst (AddL src1 src2));
9134
9135 ins_cost(INSN_COST);
9136 format %{ "add $dst, $src1, $src2" %}
9137
9138 ins_encode %{
9139 __ add(as_Register($dst$$reg),
9140 as_Register($src1$$reg),
9141 as_Register($src2$$reg));
9142 %}
9143
9144 ins_pipe(ialu_reg_reg);
9145 %}
9146
9147 // No constant pool entries requiredLong Immediate Addition.
9148 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9149 match(Set dst (AddL src1 src2));
9150
9151 ins_cost(INSN_COST);
9152 format %{ "add $dst, $src1, $src2" %}
9153
9154 // use opcode to indicate that this is an add not a sub
9155 opcode(0x0);
9156
9157 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9158
9159 ins_pipe(ialu_reg_imm);
9160 %}
9161
9162 // Integer Subtraction
9163 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9164 match(Set dst (SubI src1 src2));
9165
9166 ins_cost(INSN_COST);
9167 format %{ "subw $dst, $src1, $src2" %}
9168
9169 ins_encode %{
9170 __ subw(as_Register($dst$$reg),
9171 as_Register($src1$$reg),
9172 as_Register($src2$$reg));
9173 %}
9174
9175 ins_pipe(ialu_reg_reg);
9176 %}
9177
9178 // Immediate Subtraction
9179 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9180 match(Set dst (SubI src1 src2));
9181
9182 ins_cost(INSN_COST);
9183 format %{ "subw $dst, $src1, $src2" %}
9184
9185 // use opcode to indicate that this is a sub not an add
9186 opcode(0x1);
9187
9188 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9189
9190 ins_pipe(ialu_reg_imm);
9191 %}
9192
9193 // Long Subtraction
9194 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9195
9196 match(Set dst (SubL src1 src2));
9197
9198 ins_cost(INSN_COST);
9199 format %{ "sub $dst, $src1, $src2" %}
9200
9201 ins_encode %{
9202 __ sub(as_Register($dst$$reg),
9203 as_Register($src1$$reg),
9204 as_Register($src2$$reg));
9205 %}
9206
9207 ins_pipe(ialu_reg_reg);
9208 %}
9209
9210 // No constant pool entries requiredLong Immediate Subtraction.
9211 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9212 match(Set dst (SubL src1 src2));
9213
9214 ins_cost(INSN_COST);
9215 format %{ "sub$dst, $src1, $src2" %}
9216
9217 // use opcode to indicate that this is a sub not an add
9218 opcode(0x1);
9219
9220 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9221
9222 ins_pipe(ialu_reg_imm);
9223 %}
9224
9225 // Integer Negation (special case for sub)
9226
9227 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9228 match(Set dst (SubI zero src));
9229
9230 ins_cost(INSN_COST);
9231 format %{ "negw $dst, $src\t# int" %}
9232
9233 ins_encode %{
9234 __ negw(as_Register($dst$$reg),
9235 as_Register($src$$reg));
9236 %}
9237
9238 ins_pipe(ialu_reg);
9239 %}
9240
9241 // Long Negation
9242
9243 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9244 match(Set dst (SubL zero src));
9245
9246 ins_cost(INSN_COST);
9247 format %{ "neg $dst, $src\t# long" %}
9248
9249 ins_encode %{
9250 __ neg(as_Register($dst$$reg),
9251 as_Register($src$$reg));
9252 %}
9253
9254 ins_pipe(ialu_reg);
9255 %}
9256
9257 // Integer Multiply
9258
9259 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9260 match(Set dst (MulI src1 src2));
9261
9262 ins_cost(INSN_COST * 3);
9263 format %{ "mulw $dst, $src1, $src2" %}
9264
9265 ins_encode %{
9266 __ mulw(as_Register($dst$$reg),
9267 as_Register($src1$$reg),
9268 as_Register($src2$$reg));
9269 %}
9270
9271 ins_pipe(imul_reg_reg);
9272 %}
9273
9274 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9275 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9276
9277 ins_cost(INSN_COST * 3);
9278 format %{ "smull $dst, $src1, $src2" %}
9279
9280 ins_encode %{
9281 __ smull(as_Register($dst$$reg),
9282 as_Register($src1$$reg),
9283 as_Register($src2$$reg));
9284 %}
9285
9286 ins_pipe(imul_reg_reg);
9287 %}
9288
9289 // Long Multiply
9290
9291 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9292 match(Set dst (MulL src1 src2));
9293
9294 ins_cost(INSN_COST * 5);
9295 format %{ "mul $dst, $src1, $src2" %}
9296
9297 ins_encode %{
9298 __ mul(as_Register($dst$$reg),
9299 as_Register($src1$$reg),
9300 as_Register($src2$$reg));
9301 %}
9302
9303 ins_pipe(lmul_reg_reg);
9304 %}
9305
9306 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9307 %{
9308 match(Set dst (MulHiL src1 src2));
9309
9310 ins_cost(INSN_COST * 7);
9311 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9312
9313 ins_encode %{
9314 __ smulh(as_Register($dst$$reg),
9315 as_Register($src1$$reg),
9316 as_Register($src2$$reg));
9317 %}
9318
9319 ins_pipe(lmul_reg_reg);
9320 %}
9321
9322 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9323 %{
9324 match(Set dst (UMulHiL src1 src2));
9325
9326 ins_cost(INSN_COST * 7);
9327 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9328
9329 ins_encode %{
9330 __ umulh(as_Register($dst$$reg),
9331 as_Register($src1$$reg),
9332 as_Register($src2$$reg));
9333 %}
9334
9335 ins_pipe(lmul_reg_reg);
9336 %}
9337
9338 // Combined Integer Multiply & Add/Sub
9339
9340 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9341 match(Set dst (AddI src3 (MulI src1 src2)));
9342
9343 ins_cost(INSN_COST * 3);
9344 format %{ "madd $dst, $src1, $src2, $src3" %}
9345
9346 ins_encode %{
9347 __ maddw(as_Register($dst$$reg),
9348 as_Register($src1$$reg),
9349 as_Register($src2$$reg),
9350 as_Register($src3$$reg));
9351 %}
9352
9353 ins_pipe(imac_reg_reg);
9354 %}
9355
9356 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9357 match(Set dst (SubI src3 (MulI src1 src2)));
9358
9359 ins_cost(INSN_COST * 3);
9360 format %{ "msub $dst, $src1, $src2, $src3" %}
9361
9362 ins_encode %{
9363 __ msubw(as_Register($dst$$reg),
9364 as_Register($src1$$reg),
9365 as_Register($src2$$reg),
9366 as_Register($src3$$reg));
9367 %}
9368
9369 ins_pipe(imac_reg_reg);
9370 %}
9371
9372 // Combined Integer Multiply & Neg
9373
9374 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9375 match(Set dst (MulI (SubI zero src1) src2));
9376
9377 ins_cost(INSN_COST * 3);
9378 format %{ "mneg $dst, $src1, $src2" %}
9379
9380 ins_encode %{
9381 __ mnegw(as_Register($dst$$reg),
9382 as_Register($src1$$reg),
9383 as_Register($src2$$reg));
9384 %}
9385
9386 ins_pipe(imac_reg_reg);
9387 %}
9388
9389 // Combined Long Multiply & Add/Sub
9390
9391 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9392 match(Set dst (AddL src3 (MulL src1 src2)));
9393
9394 ins_cost(INSN_COST * 5);
9395 format %{ "madd $dst, $src1, $src2, $src3" %}
9396
9397 ins_encode %{
9398 __ madd(as_Register($dst$$reg),
9399 as_Register($src1$$reg),
9400 as_Register($src2$$reg),
9401 as_Register($src3$$reg));
9402 %}
9403
9404 ins_pipe(lmac_reg_reg);
9405 %}
9406
9407 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9408 match(Set dst (SubL src3 (MulL src1 src2)));
9409
9410 ins_cost(INSN_COST * 5);
9411 format %{ "msub $dst, $src1, $src2, $src3" %}
9412
9413 ins_encode %{
9414 __ msub(as_Register($dst$$reg),
9415 as_Register($src1$$reg),
9416 as_Register($src2$$reg),
9417 as_Register($src3$$reg));
9418 %}
9419
9420 ins_pipe(lmac_reg_reg);
9421 %}
9422
9423 // Combined Long Multiply & Neg
9424
9425 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9426 match(Set dst (MulL (SubL zero src1) src2));
9427
9428 ins_cost(INSN_COST * 5);
9429 format %{ "mneg $dst, $src1, $src2" %}
9430
9431 ins_encode %{
9432 __ mneg(as_Register($dst$$reg),
9433 as_Register($src1$$reg),
9434 as_Register($src2$$reg));
9435 %}
9436
9437 ins_pipe(lmac_reg_reg);
9438 %}
9439
9440 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9441
9442 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9443 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9444
9445 ins_cost(INSN_COST * 3);
9446 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9447
9448 ins_encode %{
9449 __ smaddl(as_Register($dst$$reg),
9450 as_Register($src1$$reg),
9451 as_Register($src2$$reg),
9452 as_Register($src3$$reg));
9453 %}
9454
9455 ins_pipe(imac_reg_reg);
9456 %}
9457
9458 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9459 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9460
9461 ins_cost(INSN_COST * 3);
9462 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9463
9464 ins_encode %{
9465 __ smsubl(as_Register($dst$$reg),
9466 as_Register($src1$$reg),
9467 as_Register($src2$$reg),
9468 as_Register($src3$$reg));
9469 %}
9470
9471 ins_pipe(imac_reg_reg);
9472 %}
9473
9474 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9475 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9476
9477 ins_cost(INSN_COST * 3);
9478 format %{ "smnegl $dst, $src1, $src2" %}
9479
9480 ins_encode %{
9481 __ smnegl(as_Register($dst$$reg),
9482 as_Register($src1$$reg),
9483 as_Register($src2$$reg));
9484 %}
9485
9486 ins_pipe(imac_reg_reg);
9487 %}
9488
9489 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9490
9491 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9492 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9493
9494 ins_cost(INSN_COST * 5);
9495 format %{ "mulw rscratch1, $src1, $src2\n\t"
9496 "maddw $dst, $src3, $src4, rscratch1" %}
9497
9498 ins_encode %{
9499 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9500 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9501
9502 ins_pipe(imac_reg_reg);
9503 %}
9504
9505 // Integer Divide
9506
9507 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9508 match(Set dst (DivI src1 src2));
9509
9510 ins_cost(INSN_COST * 19);
9511 format %{ "sdivw $dst, $src1, $src2" %}
9512
9513 ins_encode(aarch64_enc_divw(dst, src1, src2));
9514 ins_pipe(idiv_reg_reg);
9515 %}
9516
9517 // Long Divide
9518
9519 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9520 match(Set dst (DivL src1 src2));
9521
9522 ins_cost(INSN_COST * 35);
9523 format %{ "sdiv $dst, $src1, $src2" %}
9524
9525 ins_encode(aarch64_enc_div(dst, src1, src2));
9526 ins_pipe(ldiv_reg_reg);
9527 %}
9528
9529 // Integer Remainder
9530
9531 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9532 match(Set dst (ModI src1 src2));
9533
9534 ins_cost(INSN_COST * 22);
9535 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9536 "msubw $dst, rscratch1, $src2, $src1" %}
9537
9538 ins_encode(aarch64_enc_modw(dst, src1, src2));
9539 ins_pipe(idiv_reg_reg);
9540 %}
9541
9542 // Long Remainder
9543
9544 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9545 match(Set dst (ModL src1 src2));
9546
9547 ins_cost(INSN_COST * 38);
9548 format %{ "sdiv rscratch1, $src1, $src2\n"
9549 "msub $dst, rscratch1, $src2, $src1" %}
9550
9551 ins_encode(aarch64_enc_mod(dst, src1, src2));
9552 ins_pipe(ldiv_reg_reg);
9553 %}
9554
9555 // Unsigned Integer Divide
9556
9557 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9558 match(Set dst (UDivI src1 src2));
9559
9560 ins_cost(INSN_COST * 19);
9561 format %{ "udivw $dst, $src1, $src2" %}
9562
9563 ins_encode %{
9564 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9565 %}
9566
9567 ins_pipe(idiv_reg_reg);
9568 %}
9569
9570 // Unsigned Long Divide
9571
9572 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9573 match(Set dst (UDivL src1 src2));
9574
9575 ins_cost(INSN_COST * 35);
9576 format %{ "udiv $dst, $src1, $src2" %}
9577
9578 ins_encode %{
9579 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9580 %}
9581
9582 ins_pipe(ldiv_reg_reg);
9583 %}
9584
9585 // Unsigned Integer Remainder
9586
9587 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9588 match(Set dst (UModI src1 src2));
9589
9590 ins_cost(INSN_COST * 22);
9591 format %{ "udivw rscratch1, $src1, $src2\n\t"
9592 "msubw $dst, rscratch1, $src2, $src1" %}
9593
9594 ins_encode %{
9595 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9596 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9597 %}
9598
9599 ins_pipe(idiv_reg_reg);
9600 %}
9601
9602 // Unsigned Long Remainder
9603
9604 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9605 match(Set dst (UModL src1 src2));
9606
9607 ins_cost(INSN_COST * 38);
9608 format %{ "udiv rscratch1, $src1, $src2\n"
9609 "msub $dst, rscratch1, $src2, $src1" %}
9610
9611 ins_encode %{
9612 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9613 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9614 %}
9615
9616 ins_pipe(ldiv_reg_reg);
9617 %}
9618
9619 // Integer Shifts
9620
9621 // Shift Left Register
9622 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9623 match(Set dst (LShiftI src1 src2));
9624
9625 ins_cost(INSN_COST * 2);
9626 format %{ "lslvw $dst, $src1, $src2" %}
9627
9628 ins_encode %{
9629 __ lslvw(as_Register($dst$$reg),
9630 as_Register($src1$$reg),
9631 as_Register($src2$$reg));
9632 %}
9633
9634 ins_pipe(ialu_reg_reg_vshift);
9635 %}
9636
9637 // Shift Left Immediate
9638 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9639 match(Set dst (LShiftI src1 src2));
9640
9641 ins_cost(INSN_COST);
9642 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9643
9644 ins_encode %{
9645 __ lslw(as_Register($dst$$reg),
9646 as_Register($src1$$reg),
9647 $src2$$constant & 0x1f);
9648 %}
9649
9650 ins_pipe(ialu_reg_shift);
9651 %}
9652
9653 // Shift Right Logical Register
9654 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9655 match(Set dst (URShiftI src1 src2));
9656
9657 ins_cost(INSN_COST * 2);
9658 format %{ "lsrvw $dst, $src1, $src2" %}
9659
9660 ins_encode %{
9661 __ lsrvw(as_Register($dst$$reg),
9662 as_Register($src1$$reg),
9663 as_Register($src2$$reg));
9664 %}
9665
9666 ins_pipe(ialu_reg_reg_vshift);
9667 %}
9668
9669 // Shift Right Logical Immediate
9670 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9671 match(Set dst (URShiftI src1 src2));
9672
9673 ins_cost(INSN_COST);
9674 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9675
9676 ins_encode %{
9677 __ lsrw(as_Register($dst$$reg),
9678 as_Register($src1$$reg),
9679 $src2$$constant & 0x1f);
9680 %}
9681
9682 ins_pipe(ialu_reg_shift);
9683 %}
9684
9685 // Shift Right Arithmetic Register
9686 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9687 match(Set dst (RShiftI src1 src2));
9688
9689 ins_cost(INSN_COST * 2);
9690 format %{ "asrvw $dst, $src1, $src2" %}
9691
9692 ins_encode %{
9693 __ asrvw(as_Register($dst$$reg),
9694 as_Register($src1$$reg),
9695 as_Register($src2$$reg));
9696 %}
9697
9698 ins_pipe(ialu_reg_reg_vshift);
9699 %}
9700
9701 // Shift Right Arithmetic Immediate
9702 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9703 match(Set dst (RShiftI src1 src2));
9704
9705 ins_cost(INSN_COST);
9706 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9707
9708 ins_encode %{
9709 __ asrw(as_Register($dst$$reg),
9710 as_Register($src1$$reg),
9711 $src2$$constant & 0x1f);
9712 %}
9713
9714 ins_pipe(ialu_reg_shift);
9715 %}
9716
9717 // Combined Int Mask and Right Shift (using UBFM)
9718 // TODO
9719
9720 // Long Shifts
9721
9722 // Shift Left Register
9723 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9724 match(Set dst (LShiftL src1 src2));
9725
9726 ins_cost(INSN_COST * 2);
9727 format %{ "lslv $dst, $src1, $src2" %}
9728
9729 ins_encode %{
9730 __ lslv(as_Register($dst$$reg),
9731 as_Register($src1$$reg),
9732 as_Register($src2$$reg));
9733 %}
9734
9735 ins_pipe(ialu_reg_reg_vshift);
9736 %}
9737
9738 // Shift Left Immediate
9739 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9740 match(Set dst (LShiftL src1 src2));
9741
9742 ins_cost(INSN_COST);
9743 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9744
9745 ins_encode %{
9746 __ lsl(as_Register($dst$$reg),
9747 as_Register($src1$$reg),
9748 $src2$$constant & 0x3f);
9749 %}
9750
9751 ins_pipe(ialu_reg_shift);
9752 %}
9753
9754 // Shift Right Logical Register
9755 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9756 match(Set dst (URShiftL src1 src2));
9757
9758 ins_cost(INSN_COST * 2);
9759 format %{ "lsrv $dst, $src1, $src2" %}
9760
9761 ins_encode %{
9762 __ lsrv(as_Register($dst$$reg),
9763 as_Register($src1$$reg),
9764 as_Register($src2$$reg));
9765 %}
9766
9767 ins_pipe(ialu_reg_reg_vshift);
9768 %}
9769
9770 // Shift Right Logical Immediate
9771 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9772 match(Set dst (URShiftL src1 src2));
9773
9774 ins_cost(INSN_COST);
9775 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9776
9777 ins_encode %{
9778 __ lsr(as_Register($dst$$reg),
9779 as_Register($src1$$reg),
9780 $src2$$constant & 0x3f);
9781 %}
9782
9783 ins_pipe(ialu_reg_shift);
9784 %}
9785
9786 // A special-case pattern for card table stores.
9787 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9788 match(Set dst (URShiftL (CastP2X src1) src2));
9789
9790 ins_cost(INSN_COST);
9791 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9792
9793 ins_encode %{
9794 __ lsr(as_Register($dst$$reg),
9795 as_Register($src1$$reg),
9796 $src2$$constant & 0x3f);
9797 %}
9798
9799 ins_pipe(ialu_reg_shift);
9800 %}
9801
9802 // Shift Right Arithmetic Register
9803 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9804 match(Set dst (RShiftL src1 src2));
9805
9806 ins_cost(INSN_COST * 2);
9807 format %{ "asrv $dst, $src1, $src2" %}
9808
9809 ins_encode %{
9810 __ asrv(as_Register($dst$$reg),
9811 as_Register($src1$$reg),
9812 as_Register($src2$$reg));
9813 %}
9814
9815 ins_pipe(ialu_reg_reg_vshift);
9816 %}
9817
9818 // Shift Right Arithmetic Immediate
9819 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9820 match(Set dst (RShiftL src1 src2));
9821
9822 ins_cost(INSN_COST);
9823 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9824
9825 ins_encode %{
9826 __ asr(as_Register($dst$$reg),
9827 as_Register($src1$$reg),
9828 $src2$$constant & 0x3f);
9829 %}
9830
9831 ins_pipe(ialu_reg_shift);
9832 %}
9833
9834 // BEGIN This section of the file is automatically generated. Do not edit --------------
9835 // This section is generated from aarch64_ad.m4
9836
9837 // This pattern is automatically generated from aarch64_ad.m4
9838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9839 instruct regL_not_reg(iRegLNoSp dst,
9840 iRegL src1, immL_M1 m1,
9841 rFlagsReg cr) %{
9842 match(Set dst (XorL src1 m1));
9843 ins_cost(INSN_COST);
9844 format %{ "eon $dst, $src1, zr" %}
9845
9846 ins_encode %{
9847 __ eon(as_Register($dst$$reg),
9848 as_Register($src1$$reg),
9849 zr,
9850 Assembler::LSL, 0);
9851 %}
9852
9853 ins_pipe(ialu_reg);
9854 %}
9855
9856 // This pattern is automatically generated from aarch64_ad.m4
9857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9858 instruct regI_not_reg(iRegINoSp dst,
9859 iRegIorL2I src1, immI_M1 m1,
9860 rFlagsReg cr) %{
9861 match(Set dst (XorI src1 m1));
9862 ins_cost(INSN_COST);
9863 format %{ "eonw $dst, $src1, zr" %}
9864
9865 ins_encode %{
9866 __ eonw(as_Register($dst$$reg),
9867 as_Register($src1$$reg),
9868 zr,
9869 Assembler::LSL, 0);
9870 %}
9871
9872 ins_pipe(ialu_reg);
9873 %}
9874
9875 // This pattern is automatically generated from aarch64_ad.m4
9876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9877 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9878 immI0 zero, iRegIorL2I src1, immI src2) %{
9879 match(Set dst (SubI zero (URShiftI src1 src2)));
9880
9881 ins_cost(1.9 * INSN_COST);
9882 format %{ "negw $dst, $src1, LSR $src2" %}
9883
9884 ins_encode %{
9885 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9886 Assembler::LSR, $src2$$constant & 0x1f);
9887 %}
9888
9889 ins_pipe(ialu_reg_shift);
9890 %}
9891
9892 // This pattern is automatically generated from aarch64_ad.m4
9893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9894 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9895 immI0 zero, iRegIorL2I src1, immI src2) %{
9896 match(Set dst (SubI zero (RShiftI src1 src2)));
9897
9898 ins_cost(1.9 * INSN_COST);
9899 format %{ "negw $dst, $src1, ASR $src2" %}
9900
9901 ins_encode %{
9902 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9903 Assembler::ASR, $src2$$constant & 0x1f);
9904 %}
9905
9906 ins_pipe(ialu_reg_shift);
9907 %}
9908
9909 // This pattern is automatically generated from aarch64_ad.m4
9910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9911 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9912 immI0 zero, iRegIorL2I src1, immI src2) %{
9913 match(Set dst (SubI zero (LShiftI src1 src2)));
9914
9915 ins_cost(1.9 * INSN_COST);
9916 format %{ "negw $dst, $src1, LSL $src2" %}
9917
9918 ins_encode %{
9919 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9920 Assembler::LSL, $src2$$constant & 0x1f);
9921 %}
9922
9923 ins_pipe(ialu_reg_shift);
9924 %}
9925
9926 // This pattern is automatically generated from aarch64_ad.m4
9927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9928 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9929 immL0 zero, iRegL src1, immI src2) %{
9930 match(Set dst (SubL zero (URShiftL src1 src2)));
9931
9932 ins_cost(1.9 * INSN_COST);
9933 format %{ "neg $dst, $src1, LSR $src2" %}
9934
9935 ins_encode %{
9936 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9937 Assembler::LSR, $src2$$constant & 0x3f);
9938 %}
9939
9940 ins_pipe(ialu_reg_shift);
9941 %}
9942
9943 // This pattern is automatically generated from aarch64_ad.m4
9944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9945 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9946 immL0 zero, iRegL src1, immI src2) %{
9947 match(Set dst (SubL zero (RShiftL src1 src2)));
9948
9949 ins_cost(1.9 * INSN_COST);
9950 format %{ "neg $dst, $src1, ASR $src2" %}
9951
9952 ins_encode %{
9953 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9954 Assembler::ASR, $src2$$constant & 0x3f);
9955 %}
9956
9957 ins_pipe(ialu_reg_shift);
9958 %}
9959
9960 // This pattern is automatically generated from aarch64_ad.m4
9961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9962 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
9963 immL0 zero, iRegL src1, immI src2) %{
9964 match(Set dst (SubL zero (LShiftL src1 src2)));
9965
9966 ins_cost(1.9 * INSN_COST);
9967 format %{ "neg $dst, $src1, LSL $src2" %}
9968
9969 ins_encode %{
9970 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9971 Assembler::LSL, $src2$$constant & 0x3f);
9972 %}
9973
9974 ins_pipe(ialu_reg_shift);
9975 %}
9976
9977 // This pattern is automatically generated from aarch64_ad.m4
9978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9979 instruct AndI_reg_not_reg(iRegINoSp dst,
9980 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
9981 match(Set dst (AndI src1 (XorI src2 m1)));
9982 ins_cost(INSN_COST);
9983 format %{ "bicw $dst, $src1, $src2" %}
9984
9985 ins_encode %{
9986 __ bicw(as_Register($dst$$reg),
9987 as_Register($src1$$reg),
9988 as_Register($src2$$reg),
9989 Assembler::LSL, 0);
9990 %}
9991
9992 ins_pipe(ialu_reg_reg);
9993 %}
9994
9995 // This pattern is automatically generated from aarch64_ad.m4
9996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9997 instruct AndL_reg_not_reg(iRegLNoSp dst,
9998 iRegL src1, iRegL src2, immL_M1 m1) %{
9999 match(Set dst (AndL src1 (XorL src2 m1)));
10000 ins_cost(INSN_COST);
10001 format %{ "bic $dst, $src1, $src2" %}
10002
10003 ins_encode %{
10004 __ bic(as_Register($dst$$reg),
10005 as_Register($src1$$reg),
10006 as_Register($src2$$reg),
10007 Assembler::LSL, 0);
10008 %}
10009
10010 ins_pipe(ialu_reg_reg);
10011 %}
10012
10013 // This pattern is automatically generated from aarch64_ad.m4
10014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10015 instruct OrI_reg_not_reg(iRegINoSp dst,
10016 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10017 match(Set dst (OrI src1 (XorI src2 m1)));
10018 ins_cost(INSN_COST);
10019 format %{ "ornw $dst, $src1, $src2" %}
10020
10021 ins_encode %{
10022 __ ornw(as_Register($dst$$reg),
10023 as_Register($src1$$reg),
10024 as_Register($src2$$reg),
10025 Assembler::LSL, 0);
10026 %}
10027
10028 ins_pipe(ialu_reg_reg);
10029 %}
10030
10031 // This pattern is automatically generated from aarch64_ad.m4
10032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10033 instruct OrL_reg_not_reg(iRegLNoSp dst,
10034 iRegL src1, iRegL src2, immL_M1 m1) %{
10035 match(Set dst (OrL src1 (XorL src2 m1)));
10036 ins_cost(INSN_COST);
10037 format %{ "orn $dst, $src1, $src2" %}
10038
10039 ins_encode %{
10040 __ orn(as_Register($dst$$reg),
10041 as_Register($src1$$reg),
10042 as_Register($src2$$reg),
10043 Assembler::LSL, 0);
10044 %}
10045
10046 ins_pipe(ialu_reg_reg);
10047 %}
10048
10049 // This pattern is automatically generated from aarch64_ad.m4
10050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10051 instruct XorI_reg_not_reg(iRegINoSp dst,
10052 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10053 match(Set dst (XorI m1 (XorI src2 src1)));
10054 ins_cost(INSN_COST);
10055 format %{ "eonw $dst, $src1, $src2" %}
10056
10057 ins_encode %{
10058 __ eonw(as_Register($dst$$reg),
10059 as_Register($src1$$reg),
10060 as_Register($src2$$reg),
10061 Assembler::LSL, 0);
10062 %}
10063
10064 ins_pipe(ialu_reg_reg);
10065 %}
10066
10067 // This pattern is automatically generated from aarch64_ad.m4
10068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10069 instruct XorL_reg_not_reg(iRegLNoSp dst,
10070 iRegL src1, iRegL src2, immL_M1 m1) %{
10071 match(Set dst (XorL m1 (XorL src2 src1)));
10072 ins_cost(INSN_COST);
10073 format %{ "eon $dst, $src1, $src2" %}
10074
10075 ins_encode %{
10076 __ eon(as_Register($dst$$reg),
10077 as_Register($src1$$reg),
10078 as_Register($src2$$reg),
10079 Assembler::LSL, 0);
10080 %}
10081
10082 ins_pipe(ialu_reg_reg);
10083 %}
10084
10085 // This pattern is automatically generated from aarch64_ad.m4
10086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10087 // val & (-1 ^ (val >>> shift)) ==> bicw
10088 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10089 iRegIorL2I src1, iRegIorL2I src2,
10090 immI src3, immI_M1 src4) %{
10091 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10092 ins_cost(1.9 * INSN_COST);
10093 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10094
10095 ins_encode %{
10096 __ bicw(as_Register($dst$$reg),
10097 as_Register($src1$$reg),
10098 as_Register($src2$$reg),
10099 Assembler::LSR,
10100 $src3$$constant & 0x1f);
10101 %}
10102
10103 ins_pipe(ialu_reg_reg_shift);
10104 %}
10105
10106 // This pattern is automatically generated from aarch64_ad.m4
10107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10108 // val & (-1 ^ (val >>> shift)) ==> bic
10109 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10110 iRegL src1, iRegL src2,
10111 immI src3, immL_M1 src4) %{
10112 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10113 ins_cost(1.9 * INSN_COST);
10114 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10115
10116 ins_encode %{
10117 __ bic(as_Register($dst$$reg),
10118 as_Register($src1$$reg),
10119 as_Register($src2$$reg),
10120 Assembler::LSR,
10121 $src3$$constant & 0x3f);
10122 %}
10123
10124 ins_pipe(ialu_reg_reg_shift);
10125 %}
10126
10127 // This pattern is automatically generated from aarch64_ad.m4
10128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10129 // val & (-1 ^ (val >> shift)) ==> bicw
10130 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10131 iRegIorL2I src1, iRegIorL2I src2,
10132 immI src3, immI_M1 src4) %{
10133 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10134 ins_cost(1.9 * INSN_COST);
10135 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10136
10137 ins_encode %{
10138 __ bicw(as_Register($dst$$reg),
10139 as_Register($src1$$reg),
10140 as_Register($src2$$reg),
10141 Assembler::ASR,
10142 $src3$$constant & 0x1f);
10143 %}
10144
10145 ins_pipe(ialu_reg_reg_shift);
10146 %}
10147
10148 // This pattern is automatically generated from aarch64_ad.m4
10149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10150 // val & (-1 ^ (val >> shift)) ==> bic
10151 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10152 iRegL src1, iRegL src2,
10153 immI src3, immL_M1 src4) %{
10154 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10155 ins_cost(1.9 * INSN_COST);
10156 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10157
10158 ins_encode %{
10159 __ bic(as_Register($dst$$reg),
10160 as_Register($src1$$reg),
10161 as_Register($src2$$reg),
10162 Assembler::ASR,
10163 $src3$$constant & 0x3f);
10164 %}
10165
10166 ins_pipe(ialu_reg_reg_shift);
10167 %}
10168
10169 // This pattern is automatically generated from aarch64_ad.m4
10170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10171 // val & (-1 ^ (val ror shift)) ==> bicw
10172 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10173 iRegIorL2I src1, iRegIorL2I src2,
10174 immI src3, immI_M1 src4) %{
10175 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10176 ins_cost(1.9 * INSN_COST);
10177 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10178
10179 ins_encode %{
10180 __ bicw(as_Register($dst$$reg),
10181 as_Register($src1$$reg),
10182 as_Register($src2$$reg),
10183 Assembler::ROR,
10184 $src3$$constant & 0x1f);
10185 %}
10186
10187 ins_pipe(ialu_reg_reg_shift);
10188 %}
10189
10190 // This pattern is automatically generated from aarch64_ad.m4
10191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10192 // val & (-1 ^ (val ror shift)) ==> bic
10193 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10194 iRegL src1, iRegL src2,
10195 immI src3, immL_M1 src4) %{
10196 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10197 ins_cost(1.9 * INSN_COST);
10198 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10199
10200 ins_encode %{
10201 __ bic(as_Register($dst$$reg),
10202 as_Register($src1$$reg),
10203 as_Register($src2$$reg),
10204 Assembler::ROR,
10205 $src3$$constant & 0x3f);
10206 %}
10207
10208 ins_pipe(ialu_reg_reg_shift);
10209 %}
10210
10211 // This pattern is automatically generated from aarch64_ad.m4
10212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10213 // val & (-1 ^ (val << shift)) ==> bicw
10214 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10215 iRegIorL2I src1, iRegIorL2I src2,
10216 immI src3, immI_M1 src4) %{
10217 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10218 ins_cost(1.9 * INSN_COST);
10219 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10220
10221 ins_encode %{
10222 __ bicw(as_Register($dst$$reg),
10223 as_Register($src1$$reg),
10224 as_Register($src2$$reg),
10225 Assembler::LSL,
10226 $src3$$constant & 0x1f);
10227 %}
10228
10229 ins_pipe(ialu_reg_reg_shift);
10230 %}
10231
10232 // This pattern is automatically generated from aarch64_ad.m4
10233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10234 // val & (-1 ^ (val << shift)) ==> bic
10235 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10236 iRegL src1, iRegL src2,
10237 immI src3, immL_M1 src4) %{
10238 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10239 ins_cost(1.9 * INSN_COST);
10240 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10241
10242 ins_encode %{
10243 __ bic(as_Register($dst$$reg),
10244 as_Register($src1$$reg),
10245 as_Register($src2$$reg),
10246 Assembler::LSL,
10247 $src3$$constant & 0x3f);
10248 %}
10249
10250 ins_pipe(ialu_reg_reg_shift);
10251 %}
10252
10253 // This pattern is automatically generated from aarch64_ad.m4
10254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10255 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10256 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10257 iRegIorL2I src1, iRegIorL2I src2,
10258 immI src3, immI_M1 src4) %{
10259 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10260 ins_cost(1.9 * INSN_COST);
10261 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10262
10263 ins_encode %{
10264 __ eonw(as_Register($dst$$reg),
10265 as_Register($src1$$reg),
10266 as_Register($src2$$reg),
10267 Assembler::LSR,
10268 $src3$$constant & 0x1f);
10269 %}
10270
10271 ins_pipe(ialu_reg_reg_shift);
10272 %}
10273
10274 // This pattern is automatically generated from aarch64_ad.m4
10275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10276 // val ^ (-1 ^ (val >>> shift)) ==> eon
10277 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10278 iRegL src1, iRegL src2,
10279 immI src3, immL_M1 src4) %{
10280 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10281 ins_cost(1.9 * INSN_COST);
10282 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10283
10284 ins_encode %{
10285 __ eon(as_Register($dst$$reg),
10286 as_Register($src1$$reg),
10287 as_Register($src2$$reg),
10288 Assembler::LSR,
10289 $src3$$constant & 0x3f);
10290 %}
10291
10292 ins_pipe(ialu_reg_reg_shift);
10293 %}
10294
10295 // This pattern is automatically generated from aarch64_ad.m4
10296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10297 // val ^ (-1 ^ (val >> shift)) ==> eonw
10298 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10299 iRegIorL2I src1, iRegIorL2I src2,
10300 immI src3, immI_M1 src4) %{
10301 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10302 ins_cost(1.9 * INSN_COST);
10303 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10304
10305 ins_encode %{
10306 __ eonw(as_Register($dst$$reg),
10307 as_Register($src1$$reg),
10308 as_Register($src2$$reg),
10309 Assembler::ASR,
10310 $src3$$constant & 0x1f);
10311 %}
10312
10313 ins_pipe(ialu_reg_reg_shift);
10314 %}
10315
10316 // This pattern is automatically generated from aarch64_ad.m4
10317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10318 // val ^ (-1 ^ (val >> shift)) ==> eon
10319 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10320 iRegL src1, iRegL src2,
10321 immI src3, immL_M1 src4) %{
10322 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10323 ins_cost(1.9 * INSN_COST);
10324 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10325
10326 ins_encode %{
10327 __ eon(as_Register($dst$$reg),
10328 as_Register($src1$$reg),
10329 as_Register($src2$$reg),
10330 Assembler::ASR,
10331 $src3$$constant & 0x3f);
10332 %}
10333
10334 ins_pipe(ialu_reg_reg_shift);
10335 %}
10336
10337 // This pattern is automatically generated from aarch64_ad.m4
10338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10339 // val ^ (-1 ^ (val ror shift)) ==> eonw
10340 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10341 iRegIorL2I src1, iRegIorL2I src2,
10342 immI src3, immI_M1 src4) %{
10343 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10344 ins_cost(1.9 * INSN_COST);
10345 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10346
10347 ins_encode %{
10348 __ eonw(as_Register($dst$$reg),
10349 as_Register($src1$$reg),
10350 as_Register($src2$$reg),
10351 Assembler::ROR,
10352 $src3$$constant & 0x1f);
10353 %}
10354
10355 ins_pipe(ialu_reg_reg_shift);
10356 %}
10357
10358 // This pattern is automatically generated from aarch64_ad.m4
10359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10360 // val ^ (-1 ^ (val ror shift)) ==> eon
10361 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10362 iRegL src1, iRegL src2,
10363 immI src3, immL_M1 src4) %{
10364 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10365 ins_cost(1.9 * INSN_COST);
10366 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10367
10368 ins_encode %{
10369 __ eon(as_Register($dst$$reg),
10370 as_Register($src1$$reg),
10371 as_Register($src2$$reg),
10372 Assembler::ROR,
10373 $src3$$constant & 0x3f);
10374 %}
10375
10376 ins_pipe(ialu_reg_reg_shift);
10377 %}
10378
10379 // This pattern is automatically generated from aarch64_ad.m4
10380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10381 // val ^ (-1 ^ (val << shift)) ==> eonw
10382 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10383 iRegIorL2I src1, iRegIorL2I src2,
10384 immI src3, immI_M1 src4) %{
10385 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10386 ins_cost(1.9 * INSN_COST);
10387 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10388
10389 ins_encode %{
10390 __ eonw(as_Register($dst$$reg),
10391 as_Register($src1$$reg),
10392 as_Register($src2$$reg),
10393 Assembler::LSL,
10394 $src3$$constant & 0x1f);
10395 %}
10396
10397 ins_pipe(ialu_reg_reg_shift);
10398 %}
10399
10400 // This pattern is automatically generated from aarch64_ad.m4
10401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10402 // val ^ (-1 ^ (val << shift)) ==> eon
10403 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10404 iRegL src1, iRegL src2,
10405 immI src3, immL_M1 src4) %{
10406 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10407 ins_cost(1.9 * INSN_COST);
10408 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10409
10410 ins_encode %{
10411 __ eon(as_Register($dst$$reg),
10412 as_Register($src1$$reg),
10413 as_Register($src2$$reg),
10414 Assembler::LSL,
10415 $src3$$constant & 0x3f);
10416 %}
10417
10418 ins_pipe(ialu_reg_reg_shift);
10419 %}
10420
10421 // This pattern is automatically generated from aarch64_ad.m4
10422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10423 // val | (-1 ^ (val >>> shift)) ==> ornw
10424 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10425 iRegIorL2I src1, iRegIorL2I src2,
10426 immI src3, immI_M1 src4) %{
10427 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10428 ins_cost(1.9 * INSN_COST);
10429 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10430
10431 ins_encode %{
10432 __ ornw(as_Register($dst$$reg),
10433 as_Register($src1$$reg),
10434 as_Register($src2$$reg),
10435 Assembler::LSR,
10436 $src3$$constant & 0x1f);
10437 %}
10438
10439 ins_pipe(ialu_reg_reg_shift);
10440 %}
10441
10442 // This pattern is automatically generated from aarch64_ad.m4
10443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10444 // val | (-1 ^ (val >>> shift)) ==> orn
10445 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10446 iRegL src1, iRegL src2,
10447 immI src3, immL_M1 src4) %{
10448 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10449 ins_cost(1.9 * INSN_COST);
10450 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10451
10452 ins_encode %{
10453 __ orn(as_Register($dst$$reg),
10454 as_Register($src1$$reg),
10455 as_Register($src2$$reg),
10456 Assembler::LSR,
10457 $src3$$constant & 0x3f);
10458 %}
10459
10460 ins_pipe(ialu_reg_reg_shift);
10461 %}
10462
10463 // This pattern is automatically generated from aarch64_ad.m4
10464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10465 // val | (-1 ^ (val >> shift)) ==> ornw
10466 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10467 iRegIorL2I src1, iRegIorL2I src2,
10468 immI src3, immI_M1 src4) %{
10469 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10470 ins_cost(1.9 * INSN_COST);
10471 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10472
10473 ins_encode %{
10474 __ ornw(as_Register($dst$$reg),
10475 as_Register($src1$$reg),
10476 as_Register($src2$$reg),
10477 Assembler::ASR,
10478 $src3$$constant & 0x1f);
10479 %}
10480
10481 ins_pipe(ialu_reg_reg_shift);
10482 %}
10483
10484 // This pattern is automatically generated from aarch64_ad.m4
10485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10486 // val | (-1 ^ (val >> shift)) ==> orn
10487 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10488 iRegL src1, iRegL src2,
10489 immI src3, immL_M1 src4) %{
10490 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10491 ins_cost(1.9 * INSN_COST);
10492 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10493
10494 ins_encode %{
10495 __ orn(as_Register($dst$$reg),
10496 as_Register($src1$$reg),
10497 as_Register($src2$$reg),
10498 Assembler::ASR,
10499 $src3$$constant & 0x3f);
10500 %}
10501
10502 ins_pipe(ialu_reg_reg_shift);
10503 %}
10504
10505 // This pattern is automatically generated from aarch64_ad.m4
10506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10507 // val | (-1 ^ (val ror shift)) ==> ornw
10508 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10509 iRegIorL2I src1, iRegIorL2I src2,
10510 immI src3, immI_M1 src4) %{
10511 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10512 ins_cost(1.9 * INSN_COST);
10513 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10514
10515 ins_encode %{
10516 __ ornw(as_Register($dst$$reg),
10517 as_Register($src1$$reg),
10518 as_Register($src2$$reg),
10519 Assembler::ROR,
10520 $src3$$constant & 0x1f);
10521 %}
10522
10523 ins_pipe(ialu_reg_reg_shift);
10524 %}
10525
10526 // This pattern is automatically generated from aarch64_ad.m4
10527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10528 // val | (-1 ^ (val ror shift)) ==> orn
10529 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10530 iRegL src1, iRegL src2,
10531 immI src3, immL_M1 src4) %{
10532 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10533 ins_cost(1.9 * INSN_COST);
10534 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10535
10536 ins_encode %{
10537 __ orn(as_Register($dst$$reg),
10538 as_Register($src1$$reg),
10539 as_Register($src2$$reg),
10540 Assembler::ROR,
10541 $src3$$constant & 0x3f);
10542 %}
10543
10544 ins_pipe(ialu_reg_reg_shift);
10545 %}
10546
10547 // This pattern is automatically generated from aarch64_ad.m4
10548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10549 // val | (-1 ^ (val << shift)) ==> ornw
10550 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10551 iRegIorL2I src1, iRegIorL2I src2,
10552 immI src3, immI_M1 src4) %{
10553 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10554 ins_cost(1.9 * INSN_COST);
10555 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10556
10557 ins_encode %{
10558 __ ornw(as_Register($dst$$reg),
10559 as_Register($src1$$reg),
10560 as_Register($src2$$reg),
10561 Assembler::LSL,
10562 $src3$$constant & 0x1f);
10563 %}
10564
10565 ins_pipe(ialu_reg_reg_shift);
10566 %}
10567
10568 // This pattern is automatically generated from aarch64_ad.m4
10569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10570 // val | (-1 ^ (val << shift)) ==> orn
10571 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10572 iRegL src1, iRegL src2,
10573 immI src3, immL_M1 src4) %{
10574 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10575 ins_cost(1.9 * INSN_COST);
10576 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10577
10578 ins_encode %{
10579 __ orn(as_Register($dst$$reg),
10580 as_Register($src1$$reg),
10581 as_Register($src2$$reg),
10582 Assembler::LSL,
10583 $src3$$constant & 0x3f);
10584 %}
10585
10586 ins_pipe(ialu_reg_reg_shift);
10587 %}
10588
10589 // This pattern is automatically generated from aarch64_ad.m4
10590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10591 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10592 iRegIorL2I src1, iRegIorL2I src2,
10593 immI src3) %{
10594 match(Set dst (AndI src1 (URShiftI src2 src3)));
10595
10596 ins_cost(1.9 * INSN_COST);
10597 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10598
10599 ins_encode %{
10600 __ andw(as_Register($dst$$reg),
10601 as_Register($src1$$reg),
10602 as_Register($src2$$reg),
10603 Assembler::LSR,
10604 $src3$$constant & 0x1f);
10605 %}
10606
10607 ins_pipe(ialu_reg_reg_shift);
10608 %}
10609
10610 // This pattern is automatically generated from aarch64_ad.m4
10611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10612 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10613 iRegL src1, iRegL src2,
10614 immI src3) %{
10615 match(Set dst (AndL src1 (URShiftL src2 src3)));
10616
10617 ins_cost(1.9 * INSN_COST);
10618 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10619
10620 ins_encode %{
10621 __ andr(as_Register($dst$$reg),
10622 as_Register($src1$$reg),
10623 as_Register($src2$$reg),
10624 Assembler::LSR,
10625 $src3$$constant & 0x3f);
10626 %}
10627
10628 ins_pipe(ialu_reg_reg_shift);
10629 %}
10630
10631 // This pattern is automatically generated from aarch64_ad.m4
10632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10633 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10634 iRegIorL2I src1, iRegIorL2I src2,
10635 immI src3) %{
10636 match(Set dst (AndI src1 (RShiftI src2 src3)));
10637
10638 ins_cost(1.9 * INSN_COST);
10639 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10640
10641 ins_encode %{
10642 __ andw(as_Register($dst$$reg),
10643 as_Register($src1$$reg),
10644 as_Register($src2$$reg),
10645 Assembler::ASR,
10646 $src3$$constant & 0x1f);
10647 %}
10648
10649 ins_pipe(ialu_reg_reg_shift);
10650 %}
10651
10652 // This pattern is automatically generated from aarch64_ad.m4
10653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10654 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10655 iRegL src1, iRegL src2,
10656 immI src3) %{
10657 match(Set dst (AndL src1 (RShiftL src2 src3)));
10658
10659 ins_cost(1.9 * INSN_COST);
10660 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10661
10662 ins_encode %{
10663 __ andr(as_Register($dst$$reg),
10664 as_Register($src1$$reg),
10665 as_Register($src2$$reg),
10666 Assembler::ASR,
10667 $src3$$constant & 0x3f);
10668 %}
10669
10670 ins_pipe(ialu_reg_reg_shift);
10671 %}
10672
10673 // This pattern is automatically generated from aarch64_ad.m4
10674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10675 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10676 iRegIorL2I src1, iRegIorL2I src2,
10677 immI src3) %{
10678 match(Set dst (AndI src1 (LShiftI src2 src3)));
10679
10680 ins_cost(1.9 * INSN_COST);
10681 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10682
10683 ins_encode %{
10684 __ andw(as_Register($dst$$reg),
10685 as_Register($src1$$reg),
10686 as_Register($src2$$reg),
10687 Assembler::LSL,
10688 $src3$$constant & 0x1f);
10689 %}
10690
10691 ins_pipe(ialu_reg_reg_shift);
10692 %}
10693
10694 // This pattern is automatically generated from aarch64_ad.m4
10695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10696 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10697 iRegL src1, iRegL src2,
10698 immI src3) %{
10699 match(Set dst (AndL src1 (LShiftL src2 src3)));
10700
10701 ins_cost(1.9 * INSN_COST);
10702 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10703
10704 ins_encode %{
10705 __ andr(as_Register($dst$$reg),
10706 as_Register($src1$$reg),
10707 as_Register($src2$$reg),
10708 Assembler::LSL,
10709 $src3$$constant & 0x3f);
10710 %}
10711
10712 ins_pipe(ialu_reg_reg_shift);
10713 %}
10714
10715 // This pattern is automatically generated from aarch64_ad.m4
10716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10717 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10718 iRegIorL2I src1, iRegIorL2I src2,
10719 immI src3) %{
10720 match(Set dst (AndI src1 (RotateRight src2 src3)));
10721
10722 ins_cost(1.9 * INSN_COST);
10723 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10724
10725 ins_encode %{
10726 __ andw(as_Register($dst$$reg),
10727 as_Register($src1$$reg),
10728 as_Register($src2$$reg),
10729 Assembler::ROR,
10730 $src3$$constant & 0x1f);
10731 %}
10732
10733 ins_pipe(ialu_reg_reg_shift);
10734 %}
10735
10736 // This pattern is automatically generated from aarch64_ad.m4
10737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10738 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10739 iRegL src1, iRegL src2,
10740 immI src3) %{
10741 match(Set dst (AndL src1 (RotateRight src2 src3)));
10742
10743 ins_cost(1.9 * INSN_COST);
10744 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10745
10746 ins_encode %{
10747 __ andr(as_Register($dst$$reg),
10748 as_Register($src1$$reg),
10749 as_Register($src2$$reg),
10750 Assembler::ROR,
10751 $src3$$constant & 0x3f);
10752 %}
10753
10754 ins_pipe(ialu_reg_reg_shift);
10755 %}
10756
10757 // This pattern is automatically generated from aarch64_ad.m4
10758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10759 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10760 iRegIorL2I src1, iRegIorL2I src2,
10761 immI src3) %{
10762 match(Set dst (XorI src1 (URShiftI src2 src3)));
10763
10764 ins_cost(1.9 * INSN_COST);
10765 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10766
10767 ins_encode %{
10768 __ eorw(as_Register($dst$$reg),
10769 as_Register($src1$$reg),
10770 as_Register($src2$$reg),
10771 Assembler::LSR,
10772 $src3$$constant & 0x1f);
10773 %}
10774
10775 ins_pipe(ialu_reg_reg_shift);
10776 %}
10777
10778 // This pattern is automatically generated from aarch64_ad.m4
10779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10780 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10781 iRegL src1, iRegL src2,
10782 immI src3) %{
10783 match(Set dst (XorL src1 (URShiftL src2 src3)));
10784
10785 ins_cost(1.9 * INSN_COST);
10786 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10787
10788 ins_encode %{
10789 __ eor(as_Register($dst$$reg),
10790 as_Register($src1$$reg),
10791 as_Register($src2$$reg),
10792 Assembler::LSR,
10793 $src3$$constant & 0x3f);
10794 %}
10795
10796 ins_pipe(ialu_reg_reg_shift);
10797 %}
10798
10799 // This pattern is automatically generated from aarch64_ad.m4
10800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10801 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10802 iRegIorL2I src1, iRegIorL2I src2,
10803 immI src3) %{
10804 match(Set dst (XorI src1 (RShiftI src2 src3)));
10805
10806 ins_cost(1.9 * INSN_COST);
10807 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10808
10809 ins_encode %{
10810 __ eorw(as_Register($dst$$reg),
10811 as_Register($src1$$reg),
10812 as_Register($src2$$reg),
10813 Assembler::ASR,
10814 $src3$$constant & 0x1f);
10815 %}
10816
10817 ins_pipe(ialu_reg_reg_shift);
10818 %}
10819
10820 // This pattern is automatically generated from aarch64_ad.m4
10821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10822 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10823 iRegL src1, iRegL src2,
10824 immI src3) %{
10825 match(Set dst (XorL src1 (RShiftL src2 src3)));
10826
10827 ins_cost(1.9 * INSN_COST);
10828 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10829
10830 ins_encode %{
10831 __ eor(as_Register($dst$$reg),
10832 as_Register($src1$$reg),
10833 as_Register($src2$$reg),
10834 Assembler::ASR,
10835 $src3$$constant & 0x3f);
10836 %}
10837
10838 ins_pipe(ialu_reg_reg_shift);
10839 %}
10840
10841 // This pattern is automatically generated from aarch64_ad.m4
10842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10843 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10844 iRegIorL2I src1, iRegIorL2I src2,
10845 immI src3) %{
10846 match(Set dst (XorI src1 (LShiftI src2 src3)));
10847
10848 ins_cost(1.9 * INSN_COST);
10849 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10850
10851 ins_encode %{
10852 __ eorw(as_Register($dst$$reg),
10853 as_Register($src1$$reg),
10854 as_Register($src2$$reg),
10855 Assembler::LSL,
10856 $src3$$constant & 0x1f);
10857 %}
10858
10859 ins_pipe(ialu_reg_reg_shift);
10860 %}
10861
10862 // This pattern is automatically generated from aarch64_ad.m4
10863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10864 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10865 iRegL src1, iRegL src2,
10866 immI src3) %{
10867 match(Set dst (XorL src1 (LShiftL src2 src3)));
10868
10869 ins_cost(1.9 * INSN_COST);
10870 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10871
10872 ins_encode %{
10873 __ eor(as_Register($dst$$reg),
10874 as_Register($src1$$reg),
10875 as_Register($src2$$reg),
10876 Assembler::LSL,
10877 $src3$$constant & 0x3f);
10878 %}
10879
10880 ins_pipe(ialu_reg_reg_shift);
10881 %}
10882
10883 // This pattern is automatically generated from aarch64_ad.m4
10884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10885 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10886 iRegIorL2I src1, iRegIorL2I src2,
10887 immI src3) %{
10888 match(Set dst (XorI src1 (RotateRight src2 src3)));
10889
10890 ins_cost(1.9 * INSN_COST);
10891 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10892
10893 ins_encode %{
10894 __ eorw(as_Register($dst$$reg),
10895 as_Register($src1$$reg),
10896 as_Register($src2$$reg),
10897 Assembler::ROR,
10898 $src3$$constant & 0x1f);
10899 %}
10900
10901 ins_pipe(ialu_reg_reg_shift);
10902 %}
10903
10904 // This pattern is automatically generated from aarch64_ad.m4
10905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10906 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10907 iRegL src1, iRegL src2,
10908 immI src3) %{
10909 match(Set dst (XorL src1 (RotateRight src2 src3)));
10910
10911 ins_cost(1.9 * INSN_COST);
10912 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10913
10914 ins_encode %{
10915 __ eor(as_Register($dst$$reg),
10916 as_Register($src1$$reg),
10917 as_Register($src2$$reg),
10918 Assembler::ROR,
10919 $src3$$constant & 0x3f);
10920 %}
10921
10922 ins_pipe(ialu_reg_reg_shift);
10923 %}
10924
10925 // This pattern is automatically generated from aarch64_ad.m4
10926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10927 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10928 iRegIorL2I src1, iRegIorL2I src2,
10929 immI src3) %{
10930 match(Set dst (OrI src1 (URShiftI src2 src3)));
10931
10932 ins_cost(1.9 * INSN_COST);
10933 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10934
10935 ins_encode %{
10936 __ orrw(as_Register($dst$$reg),
10937 as_Register($src1$$reg),
10938 as_Register($src2$$reg),
10939 Assembler::LSR,
10940 $src3$$constant & 0x1f);
10941 %}
10942
10943 ins_pipe(ialu_reg_reg_shift);
10944 %}
10945
10946 // This pattern is automatically generated from aarch64_ad.m4
10947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10948 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10949 iRegL src1, iRegL src2,
10950 immI src3) %{
10951 match(Set dst (OrL src1 (URShiftL src2 src3)));
10952
10953 ins_cost(1.9 * INSN_COST);
10954 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10955
10956 ins_encode %{
10957 __ orr(as_Register($dst$$reg),
10958 as_Register($src1$$reg),
10959 as_Register($src2$$reg),
10960 Assembler::LSR,
10961 $src3$$constant & 0x3f);
10962 %}
10963
10964 ins_pipe(ialu_reg_reg_shift);
10965 %}
10966
10967 // This pattern is automatically generated from aarch64_ad.m4
10968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10969 instruct OrI_reg_RShift_reg(iRegINoSp dst,
10970 iRegIorL2I src1, iRegIorL2I src2,
10971 immI src3) %{
10972 match(Set dst (OrI src1 (RShiftI src2 src3)));
10973
10974 ins_cost(1.9 * INSN_COST);
10975 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
10976
10977 ins_encode %{
10978 __ orrw(as_Register($dst$$reg),
10979 as_Register($src1$$reg),
10980 as_Register($src2$$reg),
10981 Assembler::ASR,
10982 $src3$$constant & 0x1f);
10983 %}
10984
10985 ins_pipe(ialu_reg_reg_shift);
10986 %}
10987
10988 // This pattern is automatically generated from aarch64_ad.m4
10989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10990 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
10991 iRegL src1, iRegL src2,
10992 immI src3) %{
10993 match(Set dst (OrL src1 (RShiftL src2 src3)));
10994
10995 ins_cost(1.9 * INSN_COST);
10996 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
10997
10998 ins_encode %{
10999 __ orr(as_Register($dst$$reg),
11000 as_Register($src1$$reg),
11001 as_Register($src2$$reg),
11002 Assembler::ASR,
11003 $src3$$constant & 0x3f);
11004 %}
11005
11006 ins_pipe(ialu_reg_reg_shift);
11007 %}
11008
11009 // This pattern is automatically generated from aarch64_ad.m4
11010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11011 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11012 iRegIorL2I src1, iRegIorL2I src2,
11013 immI src3) %{
11014 match(Set dst (OrI src1 (LShiftI src2 src3)));
11015
11016 ins_cost(1.9 * INSN_COST);
11017 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11018
11019 ins_encode %{
11020 __ orrw(as_Register($dst$$reg),
11021 as_Register($src1$$reg),
11022 as_Register($src2$$reg),
11023 Assembler::LSL,
11024 $src3$$constant & 0x1f);
11025 %}
11026
11027 ins_pipe(ialu_reg_reg_shift);
11028 %}
11029
11030 // This pattern is automatically generated from aarch64_ad.m4
11031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11032 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11033 iRegL src1, iRegL src2,
11034 immI src3) %{
11035 match(Set dst (OrL src1 (LShiftL src2 src3)));
11036
11037 ins_cost(1.9 * INSN_COST);
11038 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11039
11040 ins_encode %{
11041 __ orr(as_Register($dst$$reg),
11042 as_Register($src1$$reg),
11043 as_Register($src2$$reg),
11044 Assembler::LSL,
11045 $src3$$constant & 0x3f);
11046 %}
11047
11048 ins_pipe(ialu_reg_reg_shift);
11049 %}
11050
11051 // This pattern is automatically generated from aarch64_ad.m4
11052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11053 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11054 iRegIorL2I src1, iRegIorL2I src2,
11055 immI src3) %{
11056 match(Set dst (OrI src1 (RotateRight src2 src3)));
11057
11058 ins_cost(1.9 * INSN_COST);
11059 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11060
11061 ins_encode %{
11062 __ orrw(as_Register($dst$$reg),
11063 as_Register($src1$$reg),
11064 as_Register($src2$$reg),
11065 Assembler::ROR,
11066 $src3$$constant & 0x1f);
11067 %}
11068
11069 ins_pipe(ialu_reg_reg_shift);
11070 %}
11071
11072 // This pattern is automatically generated from aarch64_ad.m4
11073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11074 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11075 iRegL src1, iRegL src2,
11076 immI src3) %{
11077 match(Set dst (OrL src1 (RotateRight src2 src3)));
11078
11079 ins_cost(1.9 * INSN_COST);
11080 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11081
11082 ins_encode %{
11083 __ orr(as_Register($dst$$reg),
11084 as_Register($src1$$reg),
11085 as_Register($src2$$reg),
11086 Assembler::ROR,
11087 $src3$$constant & 0x3f);
11088 %}
11089
11090 ins_pipe(ialu_reg_reg_shift);
11091 %}
11092
11093 // This pattern is automatically generated from aarch64_ad.m4
11094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11095 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11096 iRegIorL2I src1, iRegIorL2I src2,
11097 immI src3) %{
11098 match(Set dst (AddI src1 (URShiftI src2 src3)));
11099
11100 ins_cost(1.9 * INSN_COST);
11101 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11102
11103 ins_encode %{
11104 __ addw(as_Register($dst$$reg),
11105 as_Register($src1$$reg),
11106 as_Register($src2$$reg),
11107 Assembler::LSR,
11108 $src3$$constant & 0x1f);
11109 %}
11110
11111 ins_pipe(ialu_reg_reg_shift);
11112 %}
11113
11114 // This pattern is automatically generated from aarch64_ad.m4
11115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11116 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11117 iRegL src1, iRegL src2,
11118 immI src3) %{
11119 match(Set dst (AddL src1 (URShiftL src2 src3)));
11120
11121 ins_cost(1.9 * INSN_COST);
11122 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11123
11124 ins_encode %{
11125 __ add(as_Register($dst$$reg),
11126 as_Register($src1$$reg),
11127 as_Register($src2$$reg),
11128 Assembler::LSR,
11129 $src3$$constant & 0x3f);
11130 %}
11131
11132 ins_pipe(ialu_reg_reg_shift);
11133 %}
11134
11135 // This pattern is automatically generated from aarch64_ad.m4
11136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11137 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11138 iRegIorL2I src1, iRegIorL2I src2,
11139 immI src3) %{
11140 match(Set dst (AddI src1 (RShiftI src2 src3)));
11141
11142 ins_cost(1.9 * INSN_COST);
11143 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11144
11145 ins_encode %{
11146 __ addw(as_Register($dst$$reg),
11147 as_Register($src1$$reg),
11148 as_Register($src2$$reg),
11149 Assembler::ASR,
11150 $src3$$constant & 0x1f);
11151 %}
11152
11153 ins_pipe(ialu_reg_reg_shift);
11154 %}
11155
11156 // This pattern is automatically generated from aarch64_ad.m4
11157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11158 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11159 iRegL src1, iRegL src2,
11160 immI src3) %{
11161 match(Set dst (AddL src1 (RShiftL src2 src3)));
11162
11163 ins_cost(1.9 * INSN_COST);
11164 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11165
11166 ins_encode %{
11167 __ add(as_Register($dst$$reg),
11168 as_Register($src1$$reg),
11169 as_Register($src2$$reg),
11170 Assembler::ASR,
11171 $src3$$constant & 0x3f);
11172 %}
11173
11174 ins_pipe(ialu_reg_reg_shift);
11175 %}
11176
11177 // This pattern is automatically generated from aarch64_ad.m4
11178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11179 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11180 iRegIorL2I src1, iRegIorL2I src2,
11181 immI src3) %{
11182 match(Set dst (AddI src1 (LShiftI src2 src3)));
11183
11184 ins_cost(1.9 * INSN_COST);
11185 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11186
11187 ins_encode %{
11188 __ addw(as_Register($dst$$reg),
11189 as_Register($src1$$reg),
11190 as_Register($src2$$reg),
11191 Assembler::LSL,
11192 $src3$$constant & 0x1f);
11193 %}
11194
11195 ins_pipe(ialu_reg_reg_shift);
11196 %}
11197
11198 // This pattern is automatically generated from aarch64_ad.m4
11199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11200 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11201 iRegL src1, iRegL src2,
11202 immI src3) %{
11203 match(Set dst (AddL src1 (LShiftL src2 src3)));
11204
11205 ins_cost(1.9 * INSN_COST);
11206 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11207
11208 ins_encode %{
11209 __ add(as_Register($dst$$reg),
11210 as_Register($src1$$reg),
11211 as_Register($src2$$reg),
11212 Assembler::LSL,
11213 $src3$$constant & 0x3f);
11214 %}
11215
11216 ins_pipe(ialu_reg_reg_shift);
11217 %}
11218
11219 // This pattern is automatically generated from aarch64_ad.m4
11220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11221 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11222 iRegIorL2I src1, iRegIorL2I src2,
11223 immI src3) %{
11224 match(Set dst (SubI src1 (URShiftI src2 src3)));
11225
11226 ins_cost(1.9 * INSN_COST);
11227 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11228
11229 ins_encode %{
11230 __ subw(as_Register($dst$$reg),
11231 as_Register($src1$$reg),
11232 as_Register($src2$$reg),
11233 Assembler::LSR,
11234 $src3$$constant & 0x1f);
11235 %}
11236
11237 ins_pipe(ialu_reg_reg_shift);
11238 %}
11239
11240 // This pattern is automatically generated from aarch64_ad.m4
11241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11242 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11243 iRegL src1, iRegL src2,
11244 immI src3) %{
11245 match(Set dst (SubL src1 (URShiftL src2 src3)));
11246
11247 ins_cost(1.9 * INSN_COST);
11248 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11249
11250 ins_encode %{
11251 __ sub(as_Register($dst$$reg),
11252 as_Register($src1$$reg),
11253 as_Register($src2$$reg),
11254 Assembler::LSR,
11255 $src3$$constant & 0x3f);
11256 %}
11257
11258 ins_pipe(ialu_reg_reg_shift);
11259 %}
11260
11261 // This pattern is automatically generated from aarch64_ad.m4
11262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11263 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11264 iRegIorL2I src1, iRegIorL2I src2,
11265 immI src3) %{
11266 match(Set dst (SubI src1 (RShiftI src2 src3)));
11267
11268 ins_cost(1.9 * INSN_COST);
11269 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11270
11271 ins_encode %{
11272 __ subw(as_Register($dst$$reg),
11273 as_Register($src1$$reg),
11274 as_Register($src2$$reg),
11275 Assembler::ASR,
11276 $src3$$constant & 0x1f);
11277 %}
11278
11279 ins_pipe(ialu_reg_reg_shift);
11280 %}
11281
11282 // This pattern is automatically generated from aarch64_ad.m4
11283 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11284 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11285 iRegL src1, iRegL src2,
11286 immI src3) %{
11287 match(Set dst (SubL src1 (RShiftL src2 src3)));
11288
11289 ins_cost(1.9 * INSN_COST);
11290 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11291
11292 ins_encode %{
11293 __ sub(as_Register($dst$$reg),
11294 as_Register($src1$$reg),
11295 as_Register($src2$$reg),
11296 Assembler::ASR,
11297 $src3$$constant & 0x3f);
11298 %}
11299
11300 ins_pipe(ialu_reg_reg_shift);
11301 %}
11302
11303 // This pattern is automatically generated from aarch64_ad.m4
11304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11305 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11306 iRegIorL2I src1, iRegIorL2I src2,
11307 immI src3) %{
11308 match(Set dst (SubI src1 (LShiftI src2 src3)));
11309
11310 ins_cost(1.9 * INSN_COST);
11311 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11312
11313 ins_encode %{
11314 __ subw(as_Register($dst$$reg),
11315 as_Register($src1$$reg),
11316 as_Register($src2$$reg),
11317 Assembler::LSL,
11318 $src3$$constant & 0x1f);
11319 %}
11320
11321 ins_pipe(ialu_reg_reg_shift);
11322 %}
11323
11324 // This pattern is automatically generated from aarch64_ad.m4
11325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11326 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11327 iRegL src1, iRegL src2,
11328 immI src3) %{
11329 match(Set dst (SubL src1 (LShiftL src2 src3)));
11330
11331 ins_cost(1.9 * INSN_COST);
11332 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11333
11334 ins_encode %{
11335 __ sub(as_Register($dst$$reg),
11336 as_Register($src1$$reg),
11337 as_Register($src2$$reg),
11338 Assembler::LSL,
11339 $src3$$constant & 0x3f);
11340 %}
11341
11342 ins_pipe(ialu_reg_reg_shift);
11343 %}
11344
11345 // This pattern is automatically generated from aarch64_ad.m4
11346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11347
11348 // Shift Left followed by Shift Right.
11349 // This idiom is used by the compiler for the i2b bytecode etc.
11350 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11351 %{
11352 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11353 ins_cost(INSN_COST * 2);
11354 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11355 ins_encode %{
11356 int lshift = $lshift_count$$constant & 63;
11357 int rshift = $rshift_count$$constant & 63;
11358 int s = 63 - lshift;
11359 int r = (rshift - lshift) & 63;
11360 __ sbfm(as_Register($dst$$reg),
11361 as_Register($src$$reg),
11362 r, s);
11363 %}
11364
11365 ins_pipe(ialu_reg_shift);
11366 %}
11367
11368 // This pattern is automatically generated from aarch64_ad.m4
11369 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11370
11371 // Shift Left followed by Shift Right.
11372 // This idiom is used by the compiler for the i2b bytecode etc.
11373 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11374 %{
11375 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11376 ins_cost(INSN_COST * 2);
11377 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11378 ins_encode %{
11379 int lshift = $lshift_count$$constant & 31;
11380 int rshift = $rshift_count$$constant & 31;
11381 int s = 31 - lshift;
11382 int r = (rshift - lshift) & 31;
11383 __ sbfmw(as_Register($dst$$reg),
11384 as_Register($src$$reg),
11385 r, s);
11386 %}
11387
11388 ins_pipe(ialu_reg_shift);
11389 %}
11390
11391 // This pattern is automatically generated from aarch64_ad.m4
11392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11393
11394 // Shift Left followed by Shift Right.
11395 // This idiom is used by the compiler for the i2b bytecode etc.
11396 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11397 %{
11398 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11399 ins_cost(INSN_COST * 2);
11400 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11401 ins_encode %{
11402 int lshift = $lshift_count$$constant & 63;
11403 int rshift = $rshift_count$$constant & 63;
11404 int s = 63 - lshift;
11405 int r = (rshift - lshift) & 63;
11406 __ ubfm(as_Register($dst$$reg),
11407 as_Register($src$$reg),
11408 r, s);
11409 %}
11410
11411 ins_pipe(ialu_reg_shift);
11412 %}
11413
11414 // This pattern is automatically generated from aarch64_ad.m4
11415 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11416
11417 // Shift Left followed by Shift Right.
11418 // This idiom is used by the compiler for the i2b bytecode etc.
11419 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11420 %{
11421 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11422 ins_cost(INSN_COST * 2);
11423 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11424 ins_encode %{
11425 int lshift = $lshift_count$$constant & 31;
11426 int rshift = $rshift_count$$constant & 31;
11427 int s = 31 - lshift;
11428 int r = (rshift - lshift) & 31;
11429 __ ubfmw(as_Register($dst$$reg),
11430 as_Register($src$$reg),
11431 r, s);
11432 %}
11433
11434 ins_pipe(ialu_reg_shift);
11435 %}
11436
11437 // Bitfield extract with shift & mask
11438
11439 // This pattern is automatically generated from aarch64_ad.m4
11440 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11441 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11442 %{
11443 match(Set dst (AndI (URShiftI src rshift) mask));
11444 // Make sure we are not going to exceed what ubfxw can do.
11445 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11446
11447 ins_cost(INSN_COST);
11448 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11449 ins_encode %{
11450 int rshift = $rshift$$constant & 31;
11451 intptr_t mask = $mask$$constant;
11452 int width = exact_log2(mask+1);
11453 __ ubfxw(as_Register($dst$$reg),
11454 as_Register($src$$reg), rshift, width);
11455 %}
11456 ins_pipe(ialu_reg_shift);
11457 %}
11458
11459 // This pattern is automatically generated from aarch64_ad.m4
11460 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11461 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11462 %{
11463 match(Set dst (AndL (URShiftL src rshift) mask));
11464 // Make sure we are not going to exceed what ubfx can do.
11465 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11466
11467 ins_cost(INSN_COST);
11468 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11469 ins_encode %{
11470 int rshift = $rshift$$constant & 63;
11471 intptr_t mask = $mask$$constant;
11472 int width = exact_log2_long(mask+1);
11473 __ ubfx(as_Register($dst$$reg),
11474 as_Register($src$$reg), rshift, width);
11475 %}
11476 ins_pipe(ialu_reg_shift);
11477 %}
11478
11479
11480 // This pattern is automatically generated from aarch64_ad.m4
11481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11482
11483 // We can use ubfx when extending an And with a mask when we know mask
11484 // is positive. We know that because immI_bitmask guarantees it.
11485 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11486 %{
11487 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11488 // Make sure we are not going to exceed what ubfxw can do.
11489 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11490
11491 ins_cost(INSN_COST * 2);
11492 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11493 ins_encode %{
11494 int rshift = $rshift$$constant & 31;
11495 intptr_t mask = $mask$$constant;
11496 int width = exact_log2(mask+1);
11497 __ ubfx(as_Register($dst$$reg),
11498 as_Register($src$$reg), rshift, width);
11499 %}
11500 ins_pipe(ialu_reg_shift);
11501 %}
11502
11503
11504 // This pattern is automatically generated from aarch64_ad.m4
11505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11506
11507 // We can use ubfiz when masking by a positive number and then left shifting the result.
11508 // We know that the mask is positive because immI_bitmask guarantees it.
11509 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11510 %{
11511 match(Set dst (LShiftI (AndI src mask) lshift));
11512 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11513
11514 ins_cost(INSN_COST);
11515 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11516 ins_encode %{
11517 int lshift = $lshift$$constant & 31;
11518 intptr_t mask = $mask$$constant;
11519 int width = exact_log2(mask+1);
11520 __ ubfizw(as_Register($dst$$reg),
11521 as_Register($src$$reg), lshift, width);
11522 %}
11523 ins_pipe(ialu_reg_shift);
11524 %}
11525
11526 // This pattern is automatically generated from aarch64_ad.m4
11527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11528
11529 // We can use ubfiz when masking by a positive number and then left shifting the result.
11530 // We know that the mask is positive because immL_bitmask guarantees it.
11531 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11532 %{
11533 match(Set dst (LShiftL (AndL src mask) lshift));
11534 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11535
11536 ins_cost(INSN_COST);
11537 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11538 ins_encode %{
11539 int lshift = $lshift$$constant & 63;
11540 intptr_t mask = $mask$$constant;
11541 int width = exact_log2_long(mask+1);
11542 __ ubfiz(as_Register($dst$$reg),
11543 as_Register($src$$reg), lshift, width);
11544 %}
11545 ins_pipe(ialu_reg_shift);
11546 %}
11547
11548 // This pattern is automatically generated from aarch64_ad.m4
11549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11550
11551 // We can use ubfiz when masking by a positive number and then left shifting the result.
11552 // We know that the mask is positive because immI_bitmask guarantees it.
11553 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11554 %{
11555 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11556 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11557
11558 ins_cost(INSN_COST);
11559 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11560 ins_encode %{
11561 int lshift = $lshift$$constant & 31;
11562 intptr_t mask = $mask$$constant;
11563 int width = exact_log2(mask+1);
11564 __ ubfizw(as_Register($dst$$reg),
11565 as_Register($src$$reg), lshift, width);
11566 %}
11567 ins_pipe(ialu_reg_shift);
11568 %}
11569
11570 // This pattern is automatically generated from aarch64_ad.m4
11571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11572
11573 // We can use ubfiz when masking by a positive number and then left shifting the result.
11574 // We know that the mask is positive because immL_bitmask guarantees it.
11575 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11576 %{
11577 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11578 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11579
11580 ins_cost(INSN_COST);
11581 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11582 ins_encode %{
11583 int lshift = $lshift$$constant & 63;
11584 intptr_t mask = $mask$$constant;
11585 int width = exact_log2_long(mask+1);
11586 __ ubfiz(as_Register($dst$$reg),
11587 as_Register($src$$reg), lshift, width);
11588 %}
11589 ins_pipe(ialu_reg_shift);
11590 %}
11591
11592
11593 // This pattern is automatically generated from aarch64_ad.m4
11594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11595
11596 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11597 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11598 %{
11599 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11600 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11601
11602 ins_cost(INSN_COST);
11603 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11604 ins_encode %{
11605 int lshift = $lshift$$constant & 63;
11606 intptr_t mask = $mask$$constant;
11607 int width = exact_log2(mask+1);
11608 __ ubfiz(as_Register($dst$$reg),
11609 as_Register($src$$reg), lshift, width);
11610 %}
11611 ins_pipe(ialu_reg_shift);
11612 %}
11613
11614 // This pattern is automatically generated from aarch64_ad.m4
11615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11616
11617 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11618 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11619 %{
11620 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11621 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11622
11623 ins_cost(INSN_COST);
11624 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11625 ins_encode %{
11626 int lshift = $lshift$$constant & 31;
11627 intptr_t mask = $mask$$constant;
11628 int width = exact_log2(mask+1);
11629 __ ubfiz(as_Register($dst$$reg),
11630 as_Register($src$$reg), lshift, width);
11631 %}
11632 ins_pipe(ialu_reg_shift);
11633 %}
11634
11635 // This pattern is automatically generated from aarch64_ad.m4
11636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11637
11638 // Can skip int2long conversions after AND with small bitmask
11639 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11640 %{
11641 match(Set dst (ConvI2L (AndI src msk)));
11642 ins_cost(INSN_COST);
11643 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11644 ins_encode %{
11645 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11646 %}
11647 ins_pipe(ialu_reg_shift);
11648 %}
11649
11650
11651 // Rotations
11652
11653 // This pattern is automatically generated from aarch64_ad.m4
11654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11655 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11656 %{
11657 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11658 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11659
11660 ins_cost(INSN_COST);
11661 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11662
11663 ins_encode %{
11664 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11665 $rshift$$constant & 63);
11666 %}
11667 ins_pipe(ialu_reg_reg_extr);
11668 %}
11669
11670
11671 // This pattern is automatically generated from aarch64_ad.m4
11672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11673 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11674 %{
11675 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11676 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11677
11678 ins_cost(INSN_COST);
11679 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11680
11681 ins_encode %{
11682 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11683 $rshift$$constant & 31);
11684 %}
11685 ins_pipe(ialu_reg_reg_extr);
11686 %}
11687
11688
11689 // This pattern is automatically generated from aarch64_ad.m4
11690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11691 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11692 %{
11693 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11694 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11695
11696 ins_cost(INSN_COST);
11697 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11698
11699 ins_encode %{
11700 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11701 $rshift$$constant & 63);
11702 %}
11703 ins_pipe(ialu_reg_reg_extr);
11704 %}
11705
11706
11707 // This pattern is automatically generated from aarch64_ad.m4
11708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11709 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11710 %{
11711 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11712 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11713
11714 ins_cost(INSN_COST);
11715 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11716
11717 ins_encode %{
11718 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11719 $rshift$$constant & 31);
11720 %}
11721 ins_pipe(ialu_reg_reg_extr);
11722 %}
11723
11724 // This pattern is automatically generated from aarch64_ad.m4
11725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11726 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11727 %{
11728 match(Set dst (RotateRight src shift));
11729
11730 ins_cost(INSN_COST);
11731 format %{ "ror $dst, $src, $shift" %}
11732
11733 ins_encode %{
11734 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11735 $shift$$constant & 0x1f);
11736 %}
11737 ins_pipe(ialu_reg_reg_vshift);
11738 %}
11739
11740 // This pattern is automatically generated from aarch64_ad.m4
11741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11742 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11743 %{
11744 match(Set dst (RotateRight src shift));
11745
11746 ins_cost(INSN_COST);
11747 format %{ "ror $dst, $src, $shift" %}
11748
11749 ins_encode %{
11750 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11751 $shift$$constant & 0x3f);
11752 %}
11753 ins_pipe(ialu_reg_reg_vshift);
11754 %}
11755
11756 // This pattern is automatically generated from aarch64_ad.m4
11757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11758 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11759 %{
11760 match(Set dst (RotateRight src shift));
11761
11762 ins_cost(INSN_COST);
11763 format %{ "ror $dst, $src, $shift" %}
11764
11765 ins_encode %{
11766 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11767 %}
11768 ins_pipe(ialu_reg_reg_vshift);
11769 %}
11770
11771 // This pattern is automatically generated from aarch64_ad.m4
11772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11773 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11774 %{
11775 match(Set dst (RotateRight src shift));
11776
11777 ins_cost(INSN_COST);
11778 format %{ "ror $dst, $src, $shift" %}
11779
11780 ins_encode %{
11781 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11782 %}
11783 ins_pipe(ialu_reg_reg_vshift);
11784 %}
11785
11786 // This pattern is automatically generated from aarch64_ad.m4
11787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11788 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11789 %{
11790 match(Set dst (RotateLeft src shift));
11791
11792 ins_cost(INSN_COST);
11793 format %{ "rol $dst, $src, $shift" %}
11794
11795 ins_encode %{
11796 __ subw(rscratch1, zr, as_Register($shift$$reg));
11797 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11798 %}
11799 ins_pipe(ialu_reg_reg_vshift);
11800 %}
11801
11802 // This pattern is automatically generated from aarch64_ad.m4
11803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11804 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11805 %{
11806 match(Set dst (RotateLeft src shift));
11807
11808 ins_cost(INSN_COST);
11809 format %{ "rol $dst, $src, $shift" %}
11810
11811 ins_encode %{
11812 __ subw(rscratch1, zr, as_Register($shift$$reg));
11813 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11814 %}
11815 ins_pipe(ialu_reg_reg_vshift);
11816 %}
11817
11818
11819 // Add/subtract (extended)
11820
11821 // This pattern is automatically generated from aarch64_ad.m4
11822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11823 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11824 %{
11825 match(Set dst (AddL src1 (ConvI2L src2)));
11826 ins_cost(INSN_COST);
11827 format %{ "add $dst, $src1, $src2, sxtw" %}
11828
11829 ins_encode %{
11830 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11831 as_Register($src2$$reg), ext::sxtw);
11832 %}
11833 ins_pipe(ialu_reg_reg);
11834 %}
11835
11836 // This pattern is automatically generated from aarch64_ad.m4
11837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11838 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11839 %{
11840 match(Set dst (SubL src1 (ConvI2L src2)));
11841 ins_cost(INSN_COST);
11842 format %{ "sub $dst, $src1, $src2, sxtw" %}
11843
11844 ins_encode %{
11845 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11846 as_Register($src2$$reg), ext::sxtw);
11847 %}
11848 ins_pipe(ialu_reg_reg);
11849 %}
11850
11851 // This pattern is automatically generated from aarch64_ad.m4
11852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11853 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11854 %{
11855 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11856 ins_cost(INSN_COST);
11857 format %{ "add $dst, $src1, $src2, sxth" %}
11858
11859 ins_encode %{
11860 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11861 as_Register($src2$$reg), ext::sxth);
11862 %}
11863 ins_pipe(ialu_reg_reg);
11864 %}
11865
11866 // This pattern is automatically generated from aarch64_ad.m4
11867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11868 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11869 %{
11870 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11871 ins_cost(INSN_COST);
11872 format %{ "add $dst, $src1, $src2, sxtb" %}
11873
11874 ins_encode %{
11875 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11876 as_Register($src2$$reg), ext::sxtb);
11877 %}
11878 ins_pipe(ialu_reg_reg);
11879 %}
11880
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11884 %{
11885 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11886 ins_cost(INSN_COST);
11887 format %{ "add $dst, $src1, $src2, uxtb" %}
11888
11889 ins_encode %{
11890 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11891 as_Register($src2$$reg), ext::uxtb);
11892 %}
11893 ins_pipe(ialu_reg_reg);
11894 %}
11895
11896 // This pattern is automatically generated from aarch64_ad.m4
11897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11898 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11899 %{
11900 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11901 ins_cost(INSN_COST);
11902 format %{ "add $dst, $src1, $src2, sxth" %}
11903
11904 ins_encode %{
11905 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11906 as_Register($src2$$reg), ext::sxth);
11907 %}
11908 ins_pipe(ialu_reg_reg);
11909 %}
11910
11911 // This pattern is automatically generated from aarch64_ad.m4
11912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11913 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11914 %{
11915 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11916 ins_cost(INSN_COST);
11917 format %{ "add $dst, $src1, $src2, sxtw" %}
11918
11919 ins_encode %{
11920 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11921 as_Register($src2$$reg), ext::sxtw);
11922 %}
11923 ins_pipe(ialu_reg_reg);
11924 %}
11925
11926 // This pattern is automatically generated from aarch64_ad.m4
11927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11928 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11929 %{
11930 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11931 ins_cost(INSN_COST);
11932 format %{ "add $dst, $src1, $src2, sxtb" %}
11933
11934 ins_encode %{
11935 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11936 as_Register($src2$$reg), ext::sxtb);
11937 %}
11938 ins_pipe(ialu_reg_reg);
11939 %}
11940
11941 // This pattern is automatically generated from aarch64_ad.m4
11942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11943 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11944 %{
11945 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11946 ins_cost(INSN_COST);
11947 format %{ "add $dst, $src1, $src2, uxtb" %}
11948
11949 ins_encode %{
11950 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11951 as_Register($src2$$reg), ext::uxtb);
11952 %}
11953 ins_pipe(ialu_reg_reg);
11954 %}
11955
11956 // This pattern is automatically generated from aarch64_ad.m4
11957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11958 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11959 %{
11960 match(Set dst (AddI src1 (AndI src2 mask)));
11961 ins_cost(INSN_COST);
11962 format %{ "addw $dst, $src1, $src2, uxtb" %}
11963
11964 ins_encode %{
11965 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11966 as_Register($src2$$reg), ext::uxtb);
11967 %}
11968 ins_pipe(ialu_reg_reg);
11969 %}
11970
11971 // This pattern is automatically generated from aarch64_ad.m4
11972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11973 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11974 %{
11975 match(Set dst (AddI src1 (AndI src2 mask)));
11976 ins_cost(INSN_COST);
11977 format %{ "addw $dst, $src1, $src2, uxth" %}
11978
11979 ins_encode %{
11980 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11981 as_Register($src2$$reg), ext::uxth);
11982 %}
11983 ins_pipe(ialu_reg_reg);
11984 %}
11985
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
11989 %{
11990 match(Set dst (AddL src1 (AndL src2 mask)));
11991 ins_cost(INSN_COST);
11992 format %{ "add $dst, $src1, $src2, uxtb" %}
11993
11994 ins_encode %{
11995 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11996 as_Register($src2$$reg), ext::uxtb);
11997 %}
11998 ins_pipe(ialu_reg_reg);
11999 %}
12000
12001 // This pattern is automatically generated from aarch64_ad.m4
12002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12003 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12004 %{
12005 match(Set dst (AddL src1 (AndL src2 mask)));
12006 ins_cost(INSN_COST);
12007 format %{ "add $dst, $src1, $src2, uxth" %}
12008
12009 ins_encode %{
12010 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12011 as_Register($src2$$reg), ext::uxth);
12012 %}
12013 ins_pipe(ialu_reg_reg);
12014 %}
12015
12016 // This pattern is automatically generated from aarch64_ad.m4
12017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12018 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12019 %{
12020 match(Set dst (AddL src1 (AndL src2 mask)));
12021 ins_cost(INSN_COST);
12022 format %{ "add $dst, $src1, $src2, uxtw" %}
12023
12024 ins_encode %{
12025 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12026 as_Register($src2$$reg), ext::uxtw);
12027 %}
12028 ins_pipe(ialu_reg_reg);
12029 %}
12030
12031 // This pattern is automatically generated from aarch64_ad.m4
12032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12033 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12034 %{
12035 match(Set dst (SubI src1 (AndI src2 mask)));
12036 ins_cost(INSN_COST);
12037 format %{ "subw $dst, $src1, $src2, uxtb" %}
12038
12039 ins_encode %{
12040 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12041 as_Register($src2$$reg), ext::uxtb);
12042 %}
12043 ins_pipe(ialu_reg_reg);
12044 %}
12045
12046 // This pattern is automatically generated from aarch64_ad.m4
12047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12048 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12049 %{
12050 match(Set dst (SubI src1 (AndI src2 mask)));
12051 ins_cost(INSN_COST);
12052 format %{ "subw $dst, $src1, $src2, uxth" %}
12053
12054 ins_encode %{
12055 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12056 as_Register($src2$$reg), ext::uxth);
12057 %}
12058 ins_pipe(ialu_reg_reg);
12059 %}
12060
12061 // This pattern is automatically generated from aarch64_ad.m4
12062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12063 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12064 %{
12065 match(Set dst (SubL src1 (AndL src2 mask)));
12066 ins_cost(INSN_COST);
12067 format %{ "sub $dst, $src1, $src2, uxtb" %}
12068
12069 ins_encode %{
12070 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12071 as_Register($src2$$reg), ext::uxtb);
12072 %}
12073 ins_pipe(ialu_reg_reg);
12074 %}
12075
12076 // This pattern is automatically generated from aarch64_ad.m4
12077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12078 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12079 %{
12080 match(Set dst (SubL src1 (AndL src2 mask)));
12081 ins_cost(INSN_COST);
12082 format %{ "sub $dst, $src1, $src2, uxth" %}
12083
12084 ins_encode %{
12085 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12086 as_Register($src2$$reg), ext::uxth);
12087 %}
12088 ins_pipe(ialu_reg_reg);
12089 %}
12090
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12094 %{
12095 match(Set dst (SubL src1 (AndL src2 mask)));
12096 ins_cost(INSN_COST);
12097 format %{ "sub $dst, $src1, $src2, uxtw" %}
12098
12099 ins_encode %{
12100 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12101 as_Register($src2$$reg), ext::uxtw);
12102 %}
12103 ins_pipe(ialu_reg_reg);
12104 %}
12105
12106
12107 // This pattern is automatically generated from aarch64_ad.m4
12108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12109 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12110 %{
12111 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12112 ins_cost(1.9 * INSN_COST);
12113 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12114
12115 ins_encode %{
12116 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12117 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12118 %}
12119 ins_pipe(ialu_reg_reg_shift);
12120 %}
12121
12122 // This pattern is automatically generated from aarch64_ad.m4
12123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12124 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12125 %{
12126 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12127 ins_cost(1.9 * INSN_COST);
12128 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12129
12130 ins_encode %{
12131 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12132 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12133 %}
12134 ins_pipe(ialu_reg_reg_shift);
12135 %}
12136
12137 // This pattern is automatically generated from aarch64_ad.m4
12138 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12139 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12140 %{
12141 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12142 ins_cost(1.9 * INSN_COST);
12143 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12144
12145 ins_encode %{
12146 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12147 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12148 %}
12149 ins_pipe(ialu_reg_reg_shift);
12150 %}
12151
12152 // This pattern is automatically generated from aarch64_ad.m4
12153 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12154 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12155 %{
12156 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12157 ins_cost(1.9 * INSN_COST);
12158 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12159
12160 ins_encode %{
12161 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12162 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12163 %}
12164 ins_pipe(ialu_reg_reg_shift);
12165 %}
12166
12167 // This pattern is automatically generated from aarch64_ad.m4
12168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12169 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12170 %{
12171 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12172 ins_cost(1.9 * INSN_COST);
12173 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12174
12175 ins_encode %{
12176 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12177 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12178 %}
12179 ins_pipe(ialu_reg_reg_shift);
12180 %}
12181
12182 // This pattern is automatically generated from aarch64_ad.m4
12183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12184 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12185 %{
12186 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12187 ins_cost(1.9 * INSN_COST);
12188 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12189
12190 ins_encode %{
12191 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12192 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12193 %}
12194 ins_pipe(ialu_reg_reg_shift);
12195 %}
12196
12197 // This pattern is automatically generated from aarch64_ad.m4
12198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12199 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12200 %{
12201 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12202 ins_cost(1.9 * INSN_COST);
12203 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12204
12205 ins_encode %{
12206 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12207 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12208 %}
12209 ins_pipe(ialu_reg_reg_shift);
12210 %}
12211
12212 // This pattern is automatically generated from aarch64_ad.m4
12213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12214 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12215 %{
12216 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12217 ins_cost(1.9 * INSN_COST);
12218 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12219
12220 ins_encode %{
12221 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12222 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12223 %}
12224 ins_pipe(ialu_reg_reg_shift);
12225 %}
12226
12227 // This pattern is automatically generated from aarch64_ad.m4
12228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12229 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12230 %{
12231 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12232 ins_cost(1.9 * INSN_COST);
12233 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12234
12235 ins_encode %{
12236 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12237 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12238 %}
12239 ins_pipe(ialu_reg_reg_shift);
12240 %}
12241
12242 // This pattern is automatically generated from aarch64_ad.m4
12243 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12244 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12245 %{
12246 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12247 ins_cost(1.9 * INSN_COST);
12248 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12249
12250 ins_encode %{
12251 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12252 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12253 %}
12254 ins_pipe(ialu_reg_reg_shift);
12255 %}
12256
12257 // This pattern is automatically generated from aarch64_ad.m4
12258 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12259 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12260 %{
12261 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12262 ins_cost(1.9 * INSN_COST);
12263 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12264
12265 ins_encode %{
12266 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12267 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12268 %}
12269 ins_pipe(ialu_reg_reg_shift);
12270 %}
12271
12272 // This pattern is automatically generated from aarch64_ad.m4
12273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12274 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12275 %{
12276 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12277 ins_cost(1.9 * INSN_COST);
12278 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12279
12280 ins_encode %{
12281 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12282 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12283 %}
12284 ins_pipe(ialu_reg_reg_shift);
12285 %}
12286
12287 // This pattern is automatically generated from aarch64_ad.m4
12288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12289 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12290 %{
12291 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12292 ins_cost(1.9 * INSN_COST);
12293 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12294
12295 ins_encode %{
12296 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12297 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12298 %}
12299 ins_pipe(ialu_reg_reg_shift);
12300 %}
12301
12302 // This pattern is automatically generated from aarch64_ad.m4
12303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12304 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12305 %{
12306 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12307 ins_cost(1.9 * INSN_COST);
12308 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12309
12310 ins_encode %{
12311 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12312 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12313 %}
12314 ins_pipe(ialu_reg_reg_shift);
12315 %}
12316
12317 // This pattern is automatically generated from aarch64_ad.m4
12318 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12319 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12320 %{
12321 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12322 ins_cost(1.9 * INSN_COST);
12323 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12324
12325 ins_encode %{
12326 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12327 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12328 %}
12329 ins_pipe(ialu_reg_reg_shift);
12330 %}
12331
12332 // This pattern is automatically generated from aarch64_ad.m4
12333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12334 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12335 %{
12336 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12337 ins_cost(1.9 * INSN_COST);
12338 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12339
12340 ins_encode %{
12341 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12342 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12343 %}
12344 ins_pipe(ialu_reg_reg_shift);
12345 %}
12346
12347 // This pattern is automatically generated from aarch64_ad.m4
12348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12349 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12350 %{
12351 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12352 ins_cost(1.9 * INSN_COST);
12353 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12354
12355 ins_encode %{
12356 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12357 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12358 %}
12359 ins_pipe(ialu_reg_reg_shift);
12360 %}
12361
12362 // This pattern is automatically generated from aarch64_ad.m4
12363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12364 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12365 %{
12366 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12367 ins_cost(1.9 * INSN_COST);
12368 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12369
12370 ins_encode %{
12371 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12372 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12373 %}
12374 ins_pipe(ialu_reg_reg_shift);
12375 %}
12376
12377 // This pattern is automatically generated from aarch64_ad.m4
12378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12379 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12380 %{
12381 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12382 ins_cost(1.9 * INSN_COST);
12383 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12384
12385 ins_encode %{
12386 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12387 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12388 %}
12389 ins_pipe(ialu_reg_reg_shift);
12390 %}
12391
12392 // This pattern is automatically generated from aarch64_ad.m4
12393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12394 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12395 %{
12396 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12397 ins_cost(1.9 * INSN_COST);
12398 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12399
12400 ins_encode %{
12401 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12402 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12403 %}
12404 ins_pipe(ialu_reg_reg_shift);
12405 %}
12406
12407 // This pattern is automatically generated from aarch64_ad.m4
12408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12409 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12410 %{
12411 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12412 ins_cost(1.9 * INSN_COST);
12413 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12414
12415 ins_encode %{
12416 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12417 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12418 %}
12419 ins_pipe(ialu_reg_reg_shift);
12420 %}
12421
12422 // This pattern is automatically generated from aarch64_ad.m4
12423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12424 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12425 %{
12426 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12427 ins_cost(1.9 * INSN_COST);
12428 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12429
12430 ins_encode %{
12431 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12432 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12433 %}
12434 ins_pipe(ialu_reg_reg_shift);
12435 %}
12436
12437 // This pattern is automatically generated from aarch64_ad.m4
12438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12439 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12440 %{
12441 effect(DEF dst, USE src1, USE src2, USE cr);
12442 ins_cost(INSN_COST * 2);
12443 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12444
12445 ins_encode %{
12446 __ cselw($dst$$Register,
12447 $src1$$Register,
12448 $src2$$Register,
12449 Assembler::LT);
12450 %}
12451 ins_pipe(icond_reg_reg);
12452 %}
12453
12454 // This pattern is automatically generated from aarch64_ad.m4
12455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12456 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12457 %{
12458 effect(DEF dst, USE src1, USE src2, USE cr);
12459 ins_cost(INSN_COST * 2);
12460 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12461
12462 ins_encode %{
12463 __ cselw($dst$$Register,
12464 $src1$$Register,
12465 $src2$$Register,
12466 Assembler::GT);
12467 %}
12468 ins_pipe(icond_reg_reg);
12469 %}
12470
12471 // This pattern is automatically generated from aarch64_ad.m4
12472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12473 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12474 %{
12475 effect(DEF dst, USE src1, USE cr);
12476 ins_cost(INSN_COST * 2);
12477 format %{ "cselw $dst, $src1, zr lt\t" %}
12478
12479 ins_encode %{
12480 __ cselw($dst$$Register,
12481 $src1$$Register,
12482 zr,
12483 Assembler::LT);
12484 %}
12485 ins_pipe(icond_reg);
12486 %}
12487
12488 // This pattern is automatically generated from aarch64_ad.m4
12489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12490 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12491 %{
12492 effect(DEF dst, USE src1, USE cr);
12493 ins_cost(INSN_COST * 2);
12494 format %{ "cselw $dst, $src1, zr gt\t" %}
12495
12496 ins_encode %{
12497 __ cselw($dst$$Register,
12498 $src1$$Register,
12499 zr,
12500 Assembler::GT);
12501 %}
12502 ins_pipe(icond_reg);
12503 %}
12504
12505 // This pattern is automatically generated from aarch64_ad.m4
12506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12507 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12508 %{
12509 effect(DEF dst, USE src1, USE cr);
12510 ins_cost(INSN_COST * 2);
12511 format %{ "csincw $dst, $src1, zr le\t" %}
12512
12513 ins_encode %{
12514 __ csincw($dst$$Register,
12515 $src1$$Register,
12516 zr,
12517 Assembler::LE);
12518 %}
12519 ins_pipe(icond_reg);
12520 %}
12521
12522 // This pattern is automatically generated from aarch64_ad.m4
12523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12524 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12525 %{
12526 effect(DEF dst, USE src1, USE cr);
12527 ins_cost(INSN_COST * 2);
12528 format %{ "csincw $dst, $src1, zr gt\t" %}
12529
12530 ins_encode %{
12531 __ csincw($dst$$Register,
12532 $src1$$Register,
12533 zr,
12534 Assembler::GT);
12535 %}
12536 ins_pipe(icond_reg);
12537 %}
12538
12539 // This pattern is automatically generated from aarch64_ad.m4
12540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12541 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12542 %{
12543 effect(DEF dst, USE src1, USE cr);
12544 ins_cost(INSN_COST * 2);
12545 format %{ "csinvw $dst, $src1, zr lt\t" %}
12546
12547 ins_encode %{
12548 __ csinvw($dst$$Register,
12549 $src1$$Register,
12550 zr,
12551 Assembler::LT);
12552 %}
12553 ins_pipe(icond_reg);
12554 %}
12555
12556 // This pattern is automatically generated from aarch64_ad.m4
12557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12558 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12559 %{
12560 effect(DEF dst, USE src1, USE cr);
12561 ins_cost(INSN_COST * 2);
12562 format %{ "csinvw $dst, $src1, zr ge\t" %}
12563
12564 ins_encode %{
12565 __ csinvw($dst$$Register,
12566 $src1$$Register,
12567 zr,
12568 Assembler::GE);
12569 %}
12570 ins_pipe(icond_reg);
12571 %}
12572
12573 // This pattern is automatically generated from aarch64_ad.m4
12574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12575 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12576 %{
12577 match(Set dst (MinI src imm));
12578 ins_cost(INSN_COST * 3);
12579 expand %{
12580 rFlagsReg cr;
12581 compI_reg_imm0(cr, src);
12582 cmovI_reg_imm0_lt(dst, src, cr);
12583 %}
12584 %}
12585
12586 // This pattern is automatically generated from aarch64_ad.m4
12587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12588 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12589 %{
12590 match(Set dst (MinI imm src));
12591 ins_cost(INSN_COST * 3);
12592 expand %{
12593 rFlagsReg cr;
12594 compI_reg_imm0(cr, src);
12595 cmovI_reg_imm0_lt(dst, src, cr);
12596 %}
12597 %}
12598
12599 // This pattern is automatically generated from aarch64_ad.m4
12600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12601 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12602 %{
12603 match(Set dst (MinI src imm));
12604 ins_cost(INSN_COST * 3);
12605 expand %{
12606 rFlagsReg cr;
12607 compI_reg_imm0(cr, src);
12608 cmovI_reg_imm1_le(dst, src, cr);
12609 %}
12610 %}
12611
12612 // This pattern is automatically generated from aarch64_ad.m4
12613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12614 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12615 %{
12616 match(Set dst (MinI imm src));
12617 ins_cost(INSN_COST * 3);
12618 expand %{
12619 rFlagsReg cr;
12620 compI_reg_imm0(cr, src);
12621 cmovI_reg_imm1_le(dst, src, cr);
12622 %}
12623 %}
12624
12625 // This pattern is automatically generated from aarch64_ad.m4
12626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12627 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12628 %{
12629 match(Set dst (MinI src imm));
12630 ins_cost(INSN_COST * 3);
12631 expand %{
12632 rFlagsReg cr;
12633 compI_reg_imm0(cr, src);
12634 cmovI_reg_immM1_lt(dst, src, cr);
12635 %}
12636 %}
12637
12638 // This pattern is automatically generated from aarch64_ad.m4
12639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12640 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12641 %{
12642 match(Set dst (MinI imm src));
12643 ins_cost(INSN_COST * 3);
12644 expand %{
12645 rFlagsReg cr;
12646 compI_reg_imm0(cr, src);
12647 cmovI_reg_immM1_lt(dst, src, cr);
12648 %}
12649 %}
12650
12651 // This pattern is automatically generated from aarch64_ad.m4
12652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12653 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12654 %{
12655 match(Set dst (MaxI src imm));
12656 ins_cost(INSN_COST * 3);
12657 expand %{
12658 rFlagsReg cr;
12659 compI_reg_imm0(cr, src);
12660 cmovI_reg_imm0_gt(dst, src, cr);
12661 %}
12662 %}
12663
12664 // This pattern is automatically generated from aarch64_ad.m4
12665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12666 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12667 %{
12668 match(Set dst (MaxI imm src));
12669 ins_cost(INSN_COST * 3);
12670 expand %{
12671 rFlagsReg cr;
12672 compI_reg_imm0(cr, src);
12673 cmovI_reg_imm0_gt(dst, src, cr);
12674 %}
12675 %}
12676
12677 // This pattern is automatically generated from aarch64_ad.m4
12678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12679 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12680 %{
12681 match(Set dst (MaxI src imm));
12682 ins_cost(INSN_COST * 3);
12683 expand %{
12684 rFlagsReg cr;
12685 compI_reg_imm0(cr, src);
12686 cmovI_reg_imm1_gt(dst, src, cr);
12687 %}
12688 %}
12689
12690 // This pattern is automatically generated from aarch64_ad.m4
12691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12692 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12693 %{
12694 match(Set dst (MaxI imm src));
12695 ins_cost(INSN_COST * 3);
12696 expand %{
12697 rFlagsReg cr;
12698 compI_reg_imm0(cr, src);
12699 cmovI_reg_imm1_gt(dst, src, cr);
12700 %}
12701 %}
12702
12703 // This pattern is automatically generated from aarch64_ad.m4
12704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12705 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12706 %{
12707 match(Set dst (MaxI src imm));
12708 ins_cost(INSN_COST * 3);
12709 expand %{
12710 rFlagsReg cr;
12711 compI_reg_imm0(cr, src);
12712 cmovI_reg_immM1_ge(dst, src, cr);
12713 %}
12714 %}
12715
12716 // This pattern is automatically generated from aarch64_ad.m4
12717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12718 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12719 %{
12720 match(Set dst (MaxI imm src));
12721 ins_cost(INSN_COST * 3);
12722 expand %{
12723 rFlagsReg cr;
12724 compI_reg_imm0(cr, src);
12725 cmovI_reg_immM1_ge(dst, src, cr);
12726 %}
12727 %}
12728
12729 // This pattern is automatically generated from aarch64_ad.m4
12730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12731 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12732 %{
12733 match(Set dst (ReverseI src));
12734 ins_cost(INSN_COST);
12735 format %{ "rbitw $dst, $src" %}
12736 ins_encode %{
12737 __ rbitw($dst$$Register, $src$$Register);
12738 %}
12739 ins_pipe(ialu_reg);
12740 %}
12741
12742 // This pattern is automatically generated from aarch64_ad.m4
12743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12744 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12745 %{
12746 match(Set dst (ReverseL src));
12747 ins_cost(INSN_COST);
12748 format %{ "rbit $dst, $src" %}
12749 ins_encode %{
12750 __ rbit($dst$$Register, $src$$Register);
12751 %}
12752 ins_pipe(ialu_reg);
12753 %}
12754
12755
12756 // END This section of the file is automatically generated. Do not edit --------------
12757
12758
12759 // ============================================================================
12760 // Floating Point Arithmetic Instructions
12761
12762 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12763 match(Set dst (AddHF src1 src2));
12764 format %{ "faddh $dst, $src1, $src2" %}
12765 ins_encode %{
12766 __ faddh($dst$$FloatRegister,
12767 $src1$$FloatRegister,
12768 $src2$$FloatRegister);
12769 %}
12770 ins_pipe(fp_dop_reg_reg_s);
12771 %}
12772
12773 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12774 match(Set dst (AddF src1 src2));
12775
12776 ins_cost(INSN_COST * 5);
12777 format %{ "fadds $dst, $src1, $src2" %}
12778
12779 ins_encode %{
12780 __ fadds(as_FloatRegister($dst$$reg),
12781 as_FloatRegister($src1$$reg),
12782 as_FloatRegister($src2$$reg));
12783 %}
12784
12785 ins_pipe(fp_dop_reg_reg_s);
12786 %}
12787
12788 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12789 match(Set dst (AddD src1 src2));
12790
12791 ins_cost(INSN_COST * 5);
12792 format %{ "faddd $dst, $src1, $src2" %}
12793
12794 ins_encode %{
12795 __ faddd(as_FloatRegister($dst$$reg),
12796 as_FloatRegister($src1$$reg),
12797 as_FloatRegister($src2$$reg));
12798 %}
12799
12800 ins_pipe(fp_dop_reg_reg_d);
12801 %}
12802
12803 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12804 match(Set dst (SubHF src1 src2));
12805 format %{ "fsubh $dst, $src1, $src2" %}
12806 ins_encode %{
12807 __ fsubh($dst$$FloatRegister,
12808 $src1$$FloatRegister,
12809 $src2$$FloatRegister);
12810 %}
12811 ins_pipe(fp_dop_reg_reg_s);
12812 %}
12813
12814 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12815 match(Set dst (SubF src1 src2));
12816
12817 ins_cost(INSN_COST * 5);
12818 format %{ "fsubs $dst, $src1, $src2" %}
12819
12820 ins_encode %{
12821 __ fsubs(as_FloatRegister($dst$$reg),
12822 as_FloatRegister($src1$$reg),
12823 as_FloatRegister($src2$$reg));
12824 %}
12825
12826 ins_pipe(fp_dop_reg_reg_s);
12827 %}
12828
12829 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12830 match(Set dst (SubD src1 src2));
12831
12832 ins_cost(INSN_COST * 5);
12833 format %{ "fsubd $dst, $src1, $src2" %}
12834
12835 ins_encode %{
12836 __ fsubd(as_FloatRegister($dst$$reg),
12837 as_FloatRegister($src1$$reg),
12838 as_FloatRegister($src2$$reg));
12839 %}
12840
12841 ins_pipe(fp_dop_reg_reg_d);
12842 %}
12843
12844 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12845 match(Set dst (MulHF src1 src2));
12846 format %{ "fmulh $dst, $src1, $src2" %}
12847 ins_encode %{
12848 __ fmulh($dst$$FloatRegister,
12849 $src1$$FloatRegister,
12850 $src2$$FloatRegister);
12851 %}
12852 ins_pipe(fp_dop_reg_reg_s);
12853 %}
12854
12855 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12856 match(Set dst (MulF src1 src2));
12857
12858 ins_cost(INSN_COST * 6);
12859 format %{ "fmuls $dst, $src1, $src2" %}
12860
12861 ins_encode %{
12862 __ fmuls(as_FloatRegister($dst$$reg),
12863 as_FloatRegister($src1$$reg),
12864 as_FloatRegister($src2$$reg));
12865 %}
12866
12867 ins_pipe(fp_dop_reg_reg_s);
12868 %}
12869
12870 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12871 match(Set dst (MulD src1 src2));
12872
12873 ins_cost(INSN_COST * 6);
12874 format %{ "fmuld $dst, $src1, $src2" %}
12875
12876 ins_encode %{
12877 __ fmuld(as_FloatRegister($dst$$reg),
12878 as_FloatRegister($src1$$reg),
12879 as_FloatRegister($src2$$reg));
12880 %}
12881
12882 ins_pipe(fp_dop_reg_reg_d);
12883 %}
12884
12885 // src1 * src2 + src3 (half-precision float)
12886 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12887 match(Set dst (FmaHF src3 (Binary src1 src2)));
12888 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12889 ins_encode %{
12890 assert(UseFMA, "Needs FMA instructions support.");
12891 __ fmaddh($dst$$FloatRegister,
12892 $src1$$FloatRegister,
12893 $src2$$FloatRegister,
12894 $src3$$FloatRegister);
12895 %}
12896 ins_pipe(pipe_class_default);
12897 %}
12898
12899 // src1 * src2 + src3
12900 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12901 match(Set dst (FmaF src3 (Binary src1 src2)));
12902
12903 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12904
12905 ins_encode %{
12906 assert(UseFMA, "Needs FMA instructions support.");
12907 __ fmadds(as_FloatRegister($dst$$reg),
12908 as_FloatRegister($src1$$reg),
12909 as_FloatRegister($src2$$reg),
12910 as_FloatRegister($src3$$reg));
12911 %}
12912
12913 ins_pipe(pipe_class_default);
12914 %}
12915
12916 // src1 * src2 + src3
12917 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12918 match(Set dst (FmaD src3 (Binary src1 src2)));
12919
12920 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12921
12922 ins_encode %{
12923 assert(UseFMA, "Needs FMA instructions support.");
12924 __ fmaddd(as_FloatRegister($dst$$reg),
12925 as_FloatRegister($src1$$reg),
12926 as_FloatRegister($src2$$reg),
12927 as_FloatRegister($src3$$reg));
12928 %}
12929
12930 ins_pipe(pipe_class_default);
12931 %}
12932
12933 // src1 * (-src2) + src3
12934 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12935 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12936 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12937
12938 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12939
12940 ins_encode %{
12941 assert(UseFMA, "Needs FMA instructions support.");
12942 __ fmsubs(as_FloatRegister($dst$$reg),
12943 as_FloatRegister($src1$$reg),
12944 as_FloatRegister($src2$$reg),
12945 as_FloatRegister($src3$$reg));
12946 %}
12947
12948 ins_pipe(pipe_class_default);
12949 %}
12950
12951 // src1 * (-src2) + src3
12952 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12953 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12954 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12955
12956 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12957
12958 ins_encode %{
12959 assert(UseFMA, "Needs FMA instructions support.");
12960 __ fmsubd(as_FloatRegister($dst$$reg),
12961 as_FloatRegister($src1$$reg),
12962 as_FloatRegister($src2$$reg),
12963 as_FloatRegister($src3$$reg));
12964 %}
12965
12966 ins_pipe(pipe_class_default);
12967 %}
12968
12969 // src1 * (-src2) - src3
12970 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
12971 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12972 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
12973
12974 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
12975
12976 ins_encode %{
12977 assert(UseFMA, "Needs FMA instructions support.");
12978 __ fnmadds(as_FloatRegister($dst$$reg),
12979 as_FloatRegister($src1$$reg),
12980 as_FloatRegister($src2$$reg),
12981 as_FloatRegister($src3$$reg));
12982 %}
12983
12984 ins_pipe(pipe_class_default);
12985 %}
12986
12987 // src1 * (-src2) - src3
12988 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
12989 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12990 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
12991
12992 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
12993
12994 ins_encode %{
12995 assert(UseFMA, "Needs FMA instructions support.");
12996 __ fnmaddd(as_FloatRegister($dst$$reg),
12997 as_FloatRegister($src1$$reg),
12998 as_FloatRegister($src2$$reg),
12999 as_FloatRegister($src3$$reg));
13000 %}
13001
13002 ins_pipe(pipe_class_default);
13003 %}
13004
13005 // src1 * src2 - src3
13006 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13007 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13008
13009 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13010
13011 ins_encode %{
13012 assert(UseFMA, "Needs FMA instructions support.");
13013 __ fnmsubs(as_FloatRegister($dst$$reg),
13014 as_FloatRegister($src1$$reg),
13015 as_FloatRegister($src2$$reg),
13016 as_FloatRegister($src3$$reg));
13017 %}
13018
13019 ins_pipe(pipe_class_default);
13020 %}
13021
13022 // src1 * src2 - src3
13023 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13024 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13025
13026 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13027
13028 ins_encode %{
13029 assert(UseFMA, "Needs FMA instructions support.");
13030 // n.b. insn name should be fnmsubd
13031 __ fnmsub(as_FloatRegister($dst$$reg),
13032 as_FloatRegister($src1$$reg),
13033 as_FloatRegister($src2$$reg),
13034 as_FloatRegister($src3$$reg));
13035 %}
13036
13037 ins_pipe(pipe_class_default);
13038 %}
13039
13040 // Math.max(HH)H (half-precision float)
13041 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13042 match(Set dst (MaxHF src1 src2));
13043 format %{ "fmaxh $dst, $src1, $src2" %}
13044 ins_encode %{
13045 __ fmaxh($dst$$FloatRegister,
13046 $src1$$FloatRegister,
13047 $src2$$FloatRegister);
13048 %}
13049 ins_pipe(fp_dop_reg_reg_s);
13050 %}
13051
13052 // Math.min(HH)H (half-precision float)
13053 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13054 match(Set dst (MinHF src1 src2));
13055 format %{ "fminh $dst, $src1, $src2" %}
13056 ins_encode %{
13057 __ fminh($dst$$FloatRegister,
13058 $src1$$FloatRegister,
13059 $src2$$FloatRegister);
13060 %}
13061 ins_pipe(fp_dop_reg_reg_s);
13062 %}
13063
13064 // Math.max(FF)F
13065 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13066 match(Set dst (MaxF src1 src2));
13067
13068 format %{ "fmaxs $dst, $src1, $src2" %}
13069 ins_encode %{
13070 __ fmaxs(as_FloatRegister($dst$$reg),
13071 as_FloatRegister($src1$$reg),
13072 as_FloatRegister($src2$$reg));
13073 %}
13074
13075 ins_pipe(fp_dop_reg_reg_s);
13076 %}
13077
13078 // Math.min(FF)F
13079 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13080 match(Set dst (MinF src1 src2));
13081
13082 format %{ "fmins $dst, $src1, $src2" %}
13083 ins_encode %{
13084 __ fmins(as_FloatRegister($dst$$reg),
13085 as_FloatRegister($src1$$reg),
13086 as_FloatRegister($src2$$reg));
13087 %}
13088
13089 ins_pipe(fp_dop_reg_reg_s);
13090 %}
13091
13092 // Math.max(DD)D
13093 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13094 match(Set dst (MaxD src1 src2));
13095
13096 format %{ "fmaxd $dst, $src1, $src2" %}
13097 ins_encode %{
13098 __ fmaxd(as_FloatRegister($dst$$reg),
13099 as_FloatRegister($src1$$reg),
13100 as_FloatRegister($src2$$reg));
13101 %}
13102
13103 ins_pipe(fp_dop_reg_reg_d);
13104 %}
13105
13106 // Math.min(DD)D
13107 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13108 match(Set dst (MinD src1 src2));
13109
13110 format %{ "fmind $dst, $src1, $src2" %}
13111 ins_encode %{
13112 __ fmind(as_FloatRegister($dst$$reg),
13113 as_FloatRegister($src1$$reg),
13114 as_FloatRegister($src2$$reg));
13115 %}
13116
13117 ins_pipe(fp_dop_reg_reg_d);
13118 %}
13119
13120 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13121 match(Set dst (DivHF src1 src2));
13122 format %{ "fdivh $dst, $src1, $src2" %}
13123 ins_encode %{
13124 __ fdivh($dst$$FloatRegister,
13125 $src1$$FloatRegister,
13126 $src2$$FloatRegister);
13127 %}
13128 ins_pipe(fp_div_s);
13129 %}
13130
13131 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13132 match(Set dst (DivF src1 src2));
13133
13134 ins_cost(INSN_COST * 18);
13135 format %{ "fdivs $dst, $src1, $src2" %}
13136
13137 ins_encode %{
13138 __ fdivs(as_FloatRegister($dst$$reg),
13139 as_FloatRegister($src1$$reg),
13140 as_FloatRegister($src2$$reg));
13141 %}
13142
13143 ins_pipe(fp_div_s);
13144 %}
13145
13146 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13147 match(Set dst (DivD src1 src2));
13148
13149 ins_cost(INSN_COST * 32);
13150 format %{ "fdivd $dst, $src1, $src2" %}
13151
13152 ins_encode %{
13153 __ fdivd(as_FloatRegister($dst$$reg),
13154 as_FloatRegister($src1$$reg),
13155 as_FloatRegister($src2$$reg));
13156 %}
13157
13158 ins_pipe(fp_div_d);
13159 %}
13160
13161 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13162 match(Set dst (NegF src));
13163
13164 ins_cost(INSN_COST * 3);
13165 format %{ "fneg $dst, $src" %}
13166
13167 ins_encode %{
13168 __ fnegs(as_FloatRegister($dst$$reg),
13169 as_FloatRegister($src$$reg));
13170 %}
13171
13172 ins_pipe(fp_uop_s);
13173 %}
13174
13175 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13176 match(Set dst (NegD src));
13177
13178 ins_cost(INSN_COST * 3);
13179 format %{ "fnegd $dst, $src" %}
13180
13181 ins_encode %{
13182 __ fnegd(as_FloatRegister($dst$$reg),
13183 as_FloatRegister($src$$reg));
13184 %}
13185
13186 ins_pipe(fp_uop_d);
13187 %}
13188
13189 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13190 %{
13191 match(Set dst (AbsI src));
13192
13193 effect(KILL cr);
13194 ins_cost(INSN_COST * 2);
13195 format %{ "cmpw $src, zr\n\t"
13196 "cnegw $dst, $src, Assembler::LT\t# int abs"
13197 %}
13198
13199 ins_encode %{
13200 __ cmpw(as_Register($src$$reg), zr);
13201 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13202 %}
13203 ins_pipe(pipe_class_default);
13204 %}
13205
13206 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13207 %{
13208 match(Set dst (AbsL src));
13209
13210 effect(KILL cr);
13211 ins_cost(INSN_COST * 2);
13212 format %{ "cmp $src, zr\n\t"
13213 "cneg $dst, $src, Assembler::LT\t# long abs"
13214 %}
13215
13216 ins_encode %{
13217 __ cmp(as_Register($src$$reg), zr);
13218 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13219 %}
13220 ins_pipe(pipe_class_default);
13221 %}
13222
13223 instruct absF_reg(vRegF dst, vRegF src) %{
13224 match(Set dst (AbsF src));
13225
13226 ins_cost(INSN_COST * 3);
13227 format %{ "fabss $dst, $src" %}
13228 ins_encode %{
13229 __ fabss(as_FloatRegister($dst$$reg),
13230 as_FloatRegister($src$$reg));
13231 %}
13232
13233 ins_pipe(fp_uop_s);
13234 %}
13235
13236 instruct absD_reg(vRegD dst, vRegD src) %{
13237 match(Set dst (AbsD src));
13238
13239 ins_cost(INSN_COST * 3);
13240 format %{ "fabsd $dst, $src" %}
13241 ins_encode %{
13242 __ fabsd(as_FloatRegister($dst$$reg),
13243 as_FloatRegister($src$$reg));
13244 %}
13245
13246 ins_pipe(fp_uop_d);
13247 %}
13248
13249 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13250 match(Set dst (AbsF (SubF src1 src2)));
13251
13252 ins_cost(INSN_COST * 3);
13253 format %{ "fabds $dst, $src1, $src2" %}
13254 ins_encode %{
13255 __ fabds(as_FloatRegister($dst$$reg),
13256 as_FloatRegister($src1$$reg),
13257 as_FloatRegister($src2$$reg));
13258 %}
13259
13260 ins_pipe(fp_uop_s);
13261 %}
13262
13263 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13264 match(Set dst (AbsD (SubD src1 src2)));
13265
13266 ins_cost(INSN_COST * 3);
13267 format %{ "fabdd $dst, $src1, $src2" %}
13268 ins_encode %{
13269 __ fabdd(as_FloatRegister($dst$$reg),
13270 as_FloatRegister($src1$$reg),
13271 as_FloatRegister($src2$$reg));
13272 %}
13273
13274 ins_pipe(fp_uop_d);
13275 %}
13276
13277 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13278 match(Set dst (SqrtD src));
13279
13280 ins_cost(INSN_COST * 50);
13281 format %{ "fsqrtd $dst, $src" %}
13282 ins_encode %{
13283 __ fsqrtd(as_FloatRegister($dst$$reg),
13284 as_FloatRegister($src$$reg));
13285 %}
13286
13287 ins_pipe(fp_div_s);
13288 %}
13289
13290 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13291 match(Set dst (SqrtF src));
13292
13293 ins_cost(INSN_COST * 50);
13294 format %{ "fsqrts $dst, $src" %}
13295 ins_encode %{
13296 __ fsqrts(as_FloatRegister($dst$$reg),
13297 as_FloatRegister($src$$reg));
13298 %}
13299
13300 ins_pipe(fp_div_d);
13301 %}
13302
13303 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13304 match(Set dst (SqrtHF src));
13305 format %{ "fsqrth $dst, $src" %}
13306 ins_encode %{
13307 __ fsqrth($dst$$FloatRegister,
13308 $src$$FloatRegister);
13309 %}
13310 ins_pipe(fp_div_s);
13311 %}
13312
13313 // Math.rint, floor, ceil
13314 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13315 match(Set dst (RoundDoubleMode src rmode));
13316 format %{ "frint $dst, $src, $rmode" %}
13317 ins_encode %{
13318 switch ($rmode$$constant) {
13319 case RoundDoubleModeNode::rmode_rint:
13320 __ frintnd(as_FloatRegister($dst$$reg),
13321 as_FloatRegister($src$$reg));
13322 break;
13323 case RoundDoubleModeNode::rmode_floor:
13324 __ frintmd(as_FloatRegister($dst$$reg),
13325 as_FloatRegister($src$$reg));
13326 break;
13327 case RoundDoubleModeNode::rmode_ceil:
13328 __ frintpd(as_FloatRegister($dst$$reg),
13329 as_FloatRegister($src$$reg));
13330 break;
13331 }
13332 %}
13333 ins_pipe(fp_uop_d);
13334 %}
13335
13336 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13337 match(Set dst (CopySignD src1 (Binary src2 zero)));
13338 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13339 format %{ "CopySignD $dst $src1 $src2" %}
13340 ins_encode %{
13341 FloatRegister dst = as_FloatRegister($dst$$reg),
13342 src1 = as_FloatRegister($src1$$reg),
13343 src2 = as_FloatRegister($src2$$reg),
13344 zero = as_FloatRegister($zero$$reg);
13345 __ fnegd(dst, zero);
13346 __ bsl(dst, __ T8B, src2, src1);
13347 %}
13348 ins_pipe(fp_uop_d);
13349 %}
13350
13351 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13352 match(Set dst (CopySignF src1 src2));
13353 effect(TEMP_DEF dst, USE src1, USE src2);
13354 format %{ "CopySignF $dst $src1 $src2" %}
13355 ins_encode %{
13356 FloatRegister dst = as_FloatRegister($dst$$reg),
13357 src1 = as_FloatRegister($src1$$reg),
13358 src2 = as_FloatRegister($src2$$reg);
13359 __ movi(dst, __ T2S, 0x80, 24);
13360 __ bsl(dst, __ T8B, src2, src1);
13361 %}
13362 ins_pipe(fp_uop_d);
13363 %}
13364
13365 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13366 match(Set dst (SignumD src (Binary zero one)));
13367 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13368 format %{ "signumD $dst, $src" %}
13369 ins_encode %{
13370 FloatRegister src = as_FloatRegister($src$$reg),
13371 dst = as_FloatRegister($dst$$reg),
13372 zero = as_FloatRegister($zero$$reg),
13373 one = as_FloatRegister($one$$reg);
13374 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13375 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13376 // Bit selection instruction gets bit from "one" for each enabled bit in
13377 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13378 // NaN the whole "src" will be copied because "dst" is zero. For all other
13379 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13380 // from "src", and all other bits are copied from 1.0.
13381 __ bsl(dst, __ T8B, one, src);
13382 %}
13383 ins_pipe(fp_uop_d);
13384 %}
13385
13386 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13387 match(Set dst (SignumF src (Binary zero one)));
13388 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13389 format %{ "signumF $dst, $src" %}
13390 ins_encode %{
13391 FloatRegister src = as_FloatRegister($src$$reg),
13392 dst = as_FloatRegister($dst$$reg),
13393 zero = as_FloatRegister($zero$$reg),
13394 one = as_FloatRegister($one$$reg);
13395 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13396 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13397 // Bit selection instruction gets bit from "one" for each enabled bit in
13398 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13399 // NaN the whole "src" will be copied because "dst" is zero. For all other
13400 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13401 // from "src", and all other bits are copied from 1.0.
13402 __ bsl(dst, __ T8B, one, src);
13403 %}
13404 ins_pipe(fp_uop_d);
13405 %}
13406
13407 instruct onspinwait() %{
13408 match(OnSpinWait);
13409 ins_cost(INSN_COST);
13410
13411 format %{ "onspinwait" %}
13412
13413 ins_encode %{
13414 __ spin_wait();
13415 %}
13416 ins_pipe(pipe_class_empty);
13417 %}
13418
13419 // ============================================================================
13420 // Logical Instructions
13421
13422 // Integer Logical Instructions
13423
13424 // And Instructions
13425
13426
13427 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13428 match(Set dst (AndI src1 src2));
13429
13430 format %{ "andw $dst, $src1, $src2\t# int" %}
13431
13432 ins_cost(INSN_COST);
13433 ins_encode %{
13434 __ andw(as_Register($dst$$reg),
13435 as_Register($src1$$reg),
13436 as_Register($src2$$reg));
13437 %}
13438
13439 ins_pipe(ialu_reg_reg);
13440 %}
13441
13442 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13443 match(Set dst (AndI src1 src2));
13444
13445 format %{ "andsw $dst, $src1, $src2\t# int" %}
13446
13447 ins_cost(INSN_COST);
13448 ins_encode %{
13449 __ andw(as_Register($dst$$reg),
13450 as_Register($src1$$reg),
13451 (uint64_t)($src2$$constant));
13452 %}
13453
13454 ins_pipe(ialu_reg_imm);
13455 %}
13456
13457 // Or Instructions
13458
13459 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13460 match(Set dst (OrI src1 src2));
13461
13462 format %{ "orrw $dst, $src1, $src2\t# int" %}
13463
13464 ins_cost(INSN_COST);
13465 ins_encode %{
13466 __ orrw(as_Register($dst$$reg),
13467 as_Register($src1$$reg),
13468 as_Register($src2$$reg));
13469 %}
13470
13471 ins_pipe(ialu_reg_reg);
13472 %}
13473
13474 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13475 match(Set dst (OrI src1 src2));
13476
13477 format %{ "orrw $dst, $src1, $src2\t# int" %}
13478
13479 ins_cost(INSN_COST);
13480 ins_encode %{
13481 __ orrw(as_Register($dst$$reg),
13482 as_Register($src1$$reg),
13483 (uint64_t)($src2$$constant));
13484 %}
13485
13486 ins_pipe(ialu_reg_imm);
13487 %}
13488
13489 // Xor Instructions
13490
13491 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13492 match(Set dst (XorI src1 src2));
13493
13494 format %{ "eorw $dst, $src1, $src2\t# int" %}
13495
13496 ins_cost(INSN_COST);
13497 ins_encode %{
13498 __ eorw(as_Register($dst$$reg),
13499 as_Register($src1$$reg),
13500 as_Register($src2$$reg));
13501 %}
13502
13503 ins_pipe(ialu_reg_reg);
13504 %}
13505
13506 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13507 match(Set dst (XorI src1 src2));
13508
13509 format %{ "eorw $dst, $src1, $src2\t# int" %}
13510
13511 ins_cost(INSN_COST);
13512 ins_encode %{
13513 __ eorw(as_Register($dst$$reg),
13514 as_Register($src1$$reg),
13515 (uint64_t)($src2$$constant));
13516 %}
13517
13518 ins_pipe(ialu_reg_imm);
13519 %}
13520
13521 // Long Logical Instructions
13522 // TODO
13523
13524 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13525 match(Set dst (AndL src1 src2));
13526
13527 format %{ "and $dst, $src1, $src2\t# int" %}
13528
13529 ins_cost(INSN_COST);
13530 ins_encode %{
13531 __ andr(as_Register($dst$$reg),
13532 as_Register($src1$$reg),
13533 as_Register($src2$$reg));
13534 %}
13535
13536 ins_pipe(ialu_reg_reg);
13537 %}
13538
13539 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13540 match(Set dst (AndL src1 src2));
13541
13542 format %{ "and $dst, $src1, $src2\t# int" %}
13543
13544 ins_cost(INSN_COST);
13545 ins_encode %{
13546 __ andr(as_Register($dst$$reg),
13547 as_Register($src1$$reg),
13548 (uint64_t)($src2$$constant));
13549 %}
13550
13551 ins_pipe(ialu_reg_imm);
13552 %}
13553
13554 // Or Instructions
13555
13556 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13557 match(Set dst (OrL src1 src2));
13558
13559 format %{ "orr $dst, $src1, $src2\t# int" %}
13560
13561 ins_cost(INSN_COST);
13562 ins_encode %{
13563 __ orr(as_Register($dst$$reg),
13564 as_Register($src1$$reg),
13565 as_Register($src2$$reg));
13566 %}
13567
13568 ins_pipe(ialu_reg_reg);
13569 %}
13570
13571 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13572 match(Set dst (OrL src1 src2));
13573
13574 format %{ "orr $dst, $src1, $src2\t# int" %}
13575
13576 ins_cost(INSN_COST);
13577 ins_encode %{
13578 __ orr(as_Register($dst$$reg),
13579 as_Register($src1$$reg),
13580 (uint64_t)($src2$$constant));
13581 %}
13582
13583 ins_pipe(ialu_reg_imm);
13584 %}
13585
13586 // Xor Instructions
13587
13588 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13589 match(Set dst (XorL src1 src2));
13590
13591 format %{ "eor $dst, $src1, $src2\t# int" %}
13592
13593 ins_cost(INSN_COST);
13594 ins_encode %{
13595 __ eor(as_Register($dst$$reg),
13596 as_Register($src1$$reg),
13597 as_Register($src2$$reg));
13598 %}
13599
13600 ins_pipe(ialu_reg_reg);
13601 %}
13602
13603 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13604 match(Set dst (XorL src1 src2));
13605
13606 ins_cost(INSN_COST);
13607 format %{ "eor $dst, $src1, $src2\t# int" %}
13608
13609 ins_encode %{
13610 __ eor(as_Register($dst$$reg),
13611 as_Register($src1$$reg),
13612 (uint64_t)($src2$$constant));
13613 %}
13614
13615 ins_pipe(ialu_reg_imm);
13616 %}
13617
13618 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13619 %{
13620 match(Set dst (ConvI2L src));
13621
13622 ins_cost(INSN_COST);
13623 format %{ "sxtw $dst, $src\t# i2l" %}
13624 ins_encode %{
13625 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13626 %}
13627 ins_pipe(ialu_reg_shift);
13628 %}
13629
13630 // this pattern occurs in bigmath arithmetic
13631 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13632 %{
13633 match(Set dst (AndL (ConvI2L src) mask));
13634
13635 ins_cost(INSN_COST);
13636 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13637 ins_encode %{
13638 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13639 %}
13640
13641 ins_pipe(ialu_reg_shift);
13642 %}
13643
13644 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13645 match(Set dst (ConvL2I src));
13646
13647 ins_cost(INSN_COST);
13648 format %{ "movw $dst, $src \t// l2i" %}
13649
13650 ins_encode %{
13651 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13652 %}
13653
13654 ins_pipe(ialu_reg);
13655 %}
13656
13657 instruct convD2F_reg(vRegF dst, vRegD src) %{
13658 match(Set dst (ConvD2F src));
13659
13660 ins_cost(INSN_COST * 5);
13661 format %{ "fcvtd $dst, $src \t// d2f" %}
13662
13663 ins_encode %{
13664 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13665 %}
13666
13667 ins_pipe(fp_d2f);
13668 %}
13669
13670 instruct convF2D_reg(vRegD dst, vRegF src) %{
13671 match(Set dst (ConvF2D src));
13672
13673 ins_cost(INSN_COST * 5);
13674 format %{ "fcvts $dst, $src \t// f2d" %}
13675
13676 ins_encode %{
13677 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13678 %}
13679
13680 ins_pipe(fp_f2d);
13681 %}
13682
13683 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13684 match(Set dst (ConvF2I src));
13685
13686 ins_cost(INSN_COST * 5);
13687 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13688
13689 ins_encode %{
13690 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13691 %}
13692
13693 ins_pipe(fp_f2i);
13694 %}
13695
13696 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13697 match(Set dst (ConvF2L src));
13698
13699 ins_cost(INSN_COST * 5);
13700 format %{ "fcvtzs $dst, $src \t// f2l" %}
13701
13702 ins_encode %{
13703 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13704 %}
13705
13706 ins_pipe(fp_f2l);
13707 %}
13708
13709 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13710 match(Set dst (ConvF2HF src));
13711 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13712 "smov $dst, $tmp\t# move result from $tmp to $dst"
13713 %}
13714 effect(TEMP tmp);
13715 ins_encode %{
13716 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13717 %}
13718 ins_pipe(pipe_slow);
13719 %}
13720
13721 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13722 match(Set dst (ConvHF2F src));
13723 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13724 "fcvt $dst, $tmp\t# convert half to single precision"
13725 %}
13726 effect(TEMP tmp);
13727 ins_encode %{
13728 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13729 %}
13730 ins_pipe(pipe_slow);
13731 %}
13732
13733 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13734 match(Set dst (ConvI2F src));
13735
13736 ins_cost(INSN_COST * 5);
13737 format %{ "scvtfws $dst, $src \t// i2f" %}
13738
13739 ins_encode %{
13740 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13741 %}
13742
13743 ins_pipe(fp_i2f);
13744 %}
13745
13746 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13747 match(Set dst (ConvL2F src));
13748
13749 ins_cost(INSN_COST * 5);
13750 format %{ "scvtfs $dst, $src \t// l2f" %}
13751
13752 ins_encode %{
13753 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13754 %}
13755
13756 ins_pipe(fp_l2f);
13757 %}
13758
13759 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13760 match(Set dst (ConvD2I src));
13761
13762 ins_cost(INSN_COST * 5);
13763 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13764
13765 ins_encode %{
13766 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13767 %}
13768
13769 ins_pipe(fp_d2i);
13770 %}
13771
13772 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13773 match(Set dst (ConvD2L src));
13774
13775 ins_cost(INSN_COST * 5);
13776 format %{ "fcvtzd $dst, $src \t// d2l" %}
13777
13778 ins_encode %{
13779 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13780 %}
13781
13782 ins_pipe(fp_d2l);
13783 %}
13784
13785 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13786 match(Set dst (ConvI2D src));
13787
13788 ins_cost(INSN_COST * 5);
13789 format %{ "scvtfwd $dst, $src \t// i2d" %}
13790
13791 ins_encode %{
13792 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13793 %}
13794
13795 ins_pipe(fp_i2d);
13796 %}
13797
13798 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13799 match(Set dst (ConvL2D src));
13800
13801 ins_cost(INSN_COST * 5);
13802 format %{ "scvtfd $dst, $src \t// l2d" %}
13803
13804 ins_encode %{
13805 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13806 %}
13807
13808 ins_pipe(fp_l2d);
13809 %}
13810
13811 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13812 %{
13813 match(Set dst (RoundD src));
13814 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13815 format %{ "java_round_double $dst,$src"%}
13816 ins_encode %{
13817 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13818 as_FloatRegister($ftmp$$reg));
13819 %}
13820 ins_pipe(pipe_slow);
13821 %}
13822
13823 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13824 %{
13825 match(Set dst (RoundF src));
13826 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13827 format %{ "java_round_float $dst,$src"%}
13828 ins_encode %{
13829 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13830 as_FloatRegister($ftmp$$reg));
13831 %}
13832 ins_pipe(pipe_slow);
13833 %}
13834
13835 // stack <-> reg and reg <-> reg shuffles with no conversion
13836
13837 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13838
13839 match(Set dst (MoveF2I src));
13840
13841 effect(DEF dst, USE src);
13842
13843 ins_cost(4 * INSN_COST);
13844
13845 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13846
13847 ins_encode %{
13848 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13849 %}
13850
13851 ins_pipe(iload_reg_reg);
13852
13853 %}
13854
13855 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13856
13857 match(Set dst (MoveI2F src));
13858
13859 effect(DEF dst, USE src);
13860
13861 ins_cost(4 * INSN_COST);
13862
13863 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13864
13865 ins_encode %{
13866 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13867 %}
13868
13869 ins_pipe(pipe_class_memory);
13870
13871 %}
13872
13873 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13874
13875 match(Set dst (MoveD2L src));
13876
13877 effect(DEF dst, USE src);
13878
13879 ins_cost(4 * INSN_COST);
13880
13881 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13882
13883 ins_encode %{
13884 __ ldr($dst$$Register, Address(sp, $src$$disp));
13885 %}
13886
13887 ins_pipe(iload_reg_reg);
13888
13889 %}
13890
13891 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13892
13893 match(Set dst (MoveL2D src));
13894
13895 effect(DEF dst, USE src);
13896
13897 ins_cost(4 * INSN_COST);
13898
13899 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13900
13901 ins_encode %{
13902 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13903 %}
13904
13905 ins_pipe(pipe_class_memory);
13906
13907 %}
13908
13909 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13910
13911 match(Set dst (MoveF2I src));
13912
13913 effect(DEF dst, USE src);
13914
13915 ins_cost(INSN_COST);
13916
13917 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13918
13919 ins_encode %{
13920 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13921 %}
13922
13923 ins_pipe(pipe_class_memory);
13924
13925 %}
13926
13927 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13928
13929 match(Set dst (MoveI2F src));
13930
13931 effect(DEF dst, USE src);
13932
13933 ins_cost(INSN_COST);
13934
13935 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13936
13937 ins_encode %{
13938 __ strw($src$$Register, Address(sp, $dst$$disp));
13939 %}
13940
13941 ins_pipe(istore_reg_reg);
13942
13943 %}
13944
13945 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13946
13947 match(Set dst (MoveD2L src));
13948
13949 effect(DEF dst, USE src);
13950
13951 ins_cost(INSN_COST);
13952
13953 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13954
13955 ins_encode %{
13956 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13957 %}
13958
13959 ins_pipe(pipe_class_memory);
13960
13961 %}
13962
13963 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13964
13965 match(Set dst (MoveL2D src));
13966
13967 effect(DEF dst, USE src);
13968
13969 ins_cost(INSN_COST);
13970
13971 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13972
13973 ins_encode %{
13974 __ str($src$$Register, Address(sp, $dst$$disp));
13975 %}
13976
13977 ins_pipe(istore_reg_reg);
13978
13979 %}
13980
13981 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13982
13983 match(Set dst (MoveF2I src));
13984
13985 effect(DEF dst, USE src);
13986
13987 ins_cost(INSN_COST);
13988
13989 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
13990
13991 ins_encode %{
13992 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
13993 %}
13994
13995 ins_pipe(fp_f2i);
13996
13997 %}
13998
13999 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14000
14001 match(Set dst (MoveI2F src));
14002
14003 effect(DEF dst, USE src);
14004
14005 ins_cost(INSN_COST);
14006
14007 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14008
14009 ins_encode %{
14010 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14011 %}
14012
14013 ins_pipe(fp_i2f);
14014
14015 %}
14016
14017 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14018
14019 match(Set dst (MoveD2L src));
14020
14021 effect(DEF dst, USE src);
14022
14023 ins_cost(INSN_COST);
14024
14025 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14026
14027 ins_encode %{
14028 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14029 %}
14030
14031 ins_pipe(fp_d2l);
14032
14033 %}
14034
14035 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14036
14037 match(Set dst (MoveL2D src));
14038
14039 effect(DEF dst, USE src);
14040
14041 ins_cost(INSN_COST);
14042
14043 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14044
14045 ins_encode %{
14046 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14047 %}
14048
14049 ins_pipe(fp_l2d);
14050
14051 %}
14052
14053 // ============================================================================
14054 // clearing of an array
14055
14056 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14057 %{
14058 match(Set dummy (ClearArray cnt base));
14059 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14060
14061 ins_cost(4 * INSN_COST);
14062 format %{ "ClearArray $cnt, $base" %}
14063
14064 ins_encode %{
14065 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14066 if (tpc == nullptr) {
14067 ciEnv::current()->record_failure("CodeCache is full");
14068 return;
14069 }
14070 %}
14071
14072 ins_pipe(pipe_class_memory);
14073 %}
14074
14075 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14076 %{
14077 predicate((uint64_t)n->in(2)->get_long()
14078 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14079 match(Set dummy (ClearArray cnt base));
14080 effect(TEMP temp, USE_KILL base, KILL cr);
14081
14082 ins_cost(4 * INSN_COST);
14083 format %{ "ClearArray $cnt, $base" %}
14084
14085 ins_encode %{
14086 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14087 if (tpc == nullptr) {
14088 ciEnv::current()->record_failure("CodeCache is full");
14089 return;
14090 }
14091 %}
14092
14093 ins_pipe(pipe_class_memory);
14094 %}
14095
14096 // ============================================================================
14097 // Overflow Math Instructions
14098
14099 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14100 %{
14101 match(Set cr (OverflowAddI op1 op2));
14102
14103 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14104 ins_cost(INSN_COST);
14105 ins_encode %{
14106 __ cmnw($op1$$Register, $op2$$Register);
14107 %}
14108
14109 ins_pipe(icmp_reg_reg);
14110 %}
14111
14112 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14113 %{
14114 match(Set cr (OverflowAddI op1 op2));
14115
14116 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14117 ins_cost(INSN_COST);
14118 ins_encode %{
14119 __ cmnw($op1$$Register, $op2$$constant);
14120 %}
14121
14122 ins_pipe(icmp_reg_imm);
14123 %}
14124
14125 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14126 %{
14127 match(Set cr (OverflowAddL op1 op2));
14128
14129 format %{ "cmn $op1, $op2\t# overflow check long" %}
14130 ins_cost(INSN_COST);
14131 ins_encode %{
14132 __ cmn($op1$$Register, $op2$$Register);
14133 %}
14134
14135 ins_pipe(icmp_reg_reg);
14136 %}
14137
14138 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14139 %{
14140 match(Set cr (OverflowAddL op1 op2));
14141
14142 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14143 ins_cost(INSN_COST);
14144 ins_encode %{
14145 __ adds(zr, $op1$$Register, $op2$$constant);
14146 %}
14147
14148 ins_pipe(icmp_reg_imm);
14149 %}
14150
14151 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14152 %{
14153 match(Set cr (OverflowSubI op1 op2));
14154
14155 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14156 ins_cost(INSN_COST);
14157 ins_encode %{
14158 __ cmpw($op1$$Register, $op2$$Register);
14159 %}
14160
14161 ins_pipe(icmp_reg_reg);
14162 %}
14163
14164 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14165 %{
14166 match(Set cr (OverflowSubI op1 op2));
14167
14168 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14169 ins_cost(INSN_COST);
14170 ins_encode %{
14171 __ cmpw($op1$$Register, $op2$$constant);
14172 %}
14173
14174 ins_pipe(icmp_reg_imm);
14175 %}
14176
14177 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14178 %{
14179 match(Set cr (OverflowSubL op1 op2));
14180
14181 format %{ "cmp $op1, $op2\t# overflow check long" %}
14182 ins_cost(INSN_COST);
14183 ins_encode %{
14184 __ cmp($op1$$Register, $op2$$Register);
14185 %}
14186
14187 ins_pipe(icmp_reg_reg);
14188 %}
14189
14190 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14191 %{
14192 match(Set cr (OverflowSubL op1 op2));
14193
14194 format %{ "cmp $op1, $op2\t# overflow check long" %}
14195 ins_cost(INSN_COST);
14196 ins_encode %{
14197 __ subs(zr, $op1$$Register, $op2$$constant);
14198 %}
14199
14200 ins_pipe(icmp_reg_imm);
14201 %}
14202
14203 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14204 %{
14205 match(Set cr (OverflowSubI zero op1));
14206
14207 format %{ "cmpw zr, $op1\t# overflow check int" %}
14208 ins_cost(INSN_COST);
14209 ins_encode %{
14210 __ cmpw(zr, $op1$$Register);
14211 %}
14212
14213 ins_pipe(icmp_reg_imm);
14214 %}
14215
14216 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14217 %{
14218 match(Set cr (OverflowSubL zero op1));
14219
14220 format %{ "cmp zr, $op1\t# overflow check long" %}
14221 ins_cost(INSN_COST);
14222 ins_encode %{
14223 __ cmp(zr, $op1$$Register);
14224 %}
14225
14226 ins_pipe(icmp_reg_imm);
14227 %}
14228
14229 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14230 %{
14231 match(Set cr (OverflowMulI op1 op2));
14232
14233 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14234 "cmp rscratch1, rscratch1, sxtw\n\t"
14235 "movw rscratch1, #0x80000000\n\t"
14236 "cselw rscratch1, rscratch1, zr, NE\n\t"
14237 "cmpw rscratch1, #1" %}
14238 ins_cost(5 * INSN_COST);
14239 ins_encode %{
14240 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14241 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14242 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14243 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14244 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14245 %}
14246
14247 ins_pipe(pipe_slow);
14248 %}
14249
14250 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14251 %{
14252 match(If cmp (OverflowMulI op1 op2));
14253 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14254 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14255 effect(USE labl, KILL cr);
14256
14257 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14258 "cmp rscratch1, rscratch1, sxtw\n\t"
14259 "b$cmp $labl" %}
14260 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14261 ins_encode %{
14262 Label* L = $labl$$label;
14263 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14264 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14265 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14266 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14267 %}
14268
14269 ins_pipe(pipe_serial);
14270 %}
14271
14272 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14273 %{
14274 match(Set cr (OverflowMulL op1 op2));
14275
14276 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14277 "smulh rscratch2, $op1, $op2\n\t"
14278 "cmp rscratch2, rscratch1, ASR #63\n\t"
14279 "movw rscratch1, #0x80000000\n\t"
14280 "cselw rscratch1, rscratch1, zr, NE\n\t"
14281 "cmpw rscratch1, #1" %}
14282 ins_cost(6 * INSN_COST);
14283 ins_encode %{
14284 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14285 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14286 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14287 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14288 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14289 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14290 %}
14291
14292 ins_pipe(pipe_slow);
14293 %}
14294
14295 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14296 %{
14297 match(If cmp (OverflowMulL op1 op2));
14298 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14299 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14300 effect(USE labl, KILL cr);
14301
14302 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14303 "smulh rscratch2, $op1, $op2\n\t"
14304 "cmp rscratch2, rscratch1, ASR #63\n\t"
14305 "b$cmp $labl" %}
14306 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14307 ins_encode %{
14308 Label* L = $labl$$label;
14309 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14310 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14311 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14312 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14313 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14314 %}
14315
14316 ins_pipe(pipe_serial);
14317 %}
14318
14319 // ============================================================================
14320 // Compare Instructions
14321
14322 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14323 %{
14324 match(Set cr (CmpI op1 op2));
14325
14326 effect(DEF cr, USE op1, USE op2);
14327
14328 ins_cost(INSN_COST);
14329 format %{ "cmpw $op1, $op2" %}
14330
14331 ins_encode(aarch64_enc_cmpw(op1, op2));
14332
14333 ins_pipe(icmp_reg_reg);
14334 %}
14335
14336 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14337 %{
14338 match(Set cr (CmpI op1 zero));
14339
14340 effect(DEF cr, USE op1);
14341
14342 ins_cost(INSN_COST);
14343 format %{ "cmpw $op1, 0" %}
14344
14345 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14346
14347 ins_pipe(icmp_reg_imm);
14348 %}
14349
14350 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14351 %{
14352 match(Set cr (CmpI op1 op2));
14353
14354 effect(DEF cr, USE op1);
14355
14356 ins_cost(INSN_COST);
14357 format %{ "cmpw $op1, $op2" %}
14358
14359 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14360
14361 ins_pipe(icmp_reg_imm);
14362 %}
14363
14364 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14365 %{
14366 match(Set cr (CmpI op1 op2));
14367
14368 effect(DEF cr, USE op1);
14369
14370 ins_cost(INSN_COST * 2);
14371 format %{ "cmpw $op1, $op2" %}
14372
14373 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14374
14375 ins_pipe(icmp_reg_imm);
14376 %}
14377
14378 // Unsigned compare Instructions; really, same as signed compare
14379 // except it should only be used to feed an If or a CMovI which takes a
14380 // cmpOpU.
14381
14382 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14383 %{
14384 match(Set cr (CmpU op1 op2));
14385
14386 effect(DEF cr, USE op1, USE op2);
14387
14388 ins_cost(INSN_COST);
14389 format %{ "cmpw $op1, $op2\t# unsigned" %}
14390
14391 ins_encode(aarch64_enc_cmpw(op1, op2));
14392
14393 ins_pipe(icmp_reg_reg);
14394 %}
14395
14396 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14397 %{
14398 match(Set cr (CmpU op1 zero));
14399
14400 effect(DEF cr, USE op1);
14401
14402 ins_cost(INSN_COST);
14403 format %{ "cmpw $op1, #0\t# unsigned" %}
14404
14405 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14406
14407 ins_pipe(icmp_reg_imm);
14408 %}
14409
14410 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14411 %{
14412 match(Set cr (CmpU op1 op2));
14413
14414 effect(DEF cr, USE op1);
14415
14416 ins_cost(INSN_COST);
14417 format %{ "cmpw $op1, $op2\t# unsigned" %}
14418
14419 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14420
14421 ins_pipe(icmp_reg_imm);
14422 %}
14423
14424 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14425 %{
14426 match(Set cr (CmpU op1 op2));
14427
14428 effect(DEF cr, USE op1);
14429
14430 ins_cost(INSN_COST * 2);
14431 format %{ "cmpw $op1, $op2\t# unsigned" %}
14432
14433 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14434
14435 ins_pipe(icmp_reg_imm);
14436 %}
14437
14438 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14439 %{
14440 match(Set cr (CmpL op1 op2));
14441
14442 effect(DEF cr, USE op1, USE op2);
14443
14444 ins_cost(INSN_COST);
14445 format %{ "cmp $op1, $op2" %}
14446
14447 ins_encode(aarch64_enc_cmp(op1, op2));
14448
14449 ins_pipe(icmp_reg_reg);
14450 %}
14451
14452 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14453 %{
14454 match(Set cr (CmpL op1 zero));
14455
14456 effect(DEF cr, USE op1);
14457
14458 ins_cost(INSN_COST);
14459 format %{ "tst $op1" %}
14460
14461 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14462
14463 ins_pipe(icmp_reg_imm);
14464 %}
14465
14466 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14467 %{
14468 match(Set cr (CmpL op1 op2));
14469
14470 effect(DEF cr, USE op1);
14471
14472 ins_cost(INSN_COST);
14473 format %{ "cmp $op1, $op2" %}
14474
14475 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14476
14477 ins_pipe(icmp_reg_imm);
14478 %}
14479
14480 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14481 %{
14482 match(Set cr (CmpL op1 op2));
14483
14484 effect(DEF cr, USE op1);
14485
14486 ins_cost(INSN_COST * 2);
14487 format %{ "cmp $op1, $op2" %}
14488
14489 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14490
14491 ins_pipe(icmp_reg_imm);
14492 %}
14493
14494 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14495 %{
14496 match(Set cr (CmpUL op1 op2));
14497
14498 effect(DEF cr, USE op1, USE op2);
14499
14500 ins_cost(INSN_COST);
14501 format %{ "cmp $op1, $op2" %}
14502
14503 ins_encode(aarch64_enc_cmp(op1, op2));
14504
14505 ins_pipe(icmp_reg_reg);
14506 %}
14507
14508 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14509 %{
14510 match(Set cr (CmpUL op1 zero));
14511
14512 effect(DEF cr, USE op1);
14513
14514 ins_cost(INSN_COST);
14515 format %{ "tst $op1" %}
14516
14517 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14518
14519 ins_pipe(icmp_reg_imm);
14520 %}
14521
14522 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14523 %{
14524 match(Set cr (CmpUL op1 op2));
14525
14526 effect(DEF cr, USE op1);
14527
14528 ins_cost(INSN_COST);
14529 format %{ "cmp $op1, $op2" %}
14530
14531 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14532
14533 ins_pipe(icmp_reg_imm);
14534 %}
14535
14536 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14537 %{
14538 match(Set cr (CmpUL op1 op2));
14539
14540 effect(DEF cr, USE op1);
14541
14542 ins_cost(INSN_COST * 2);
14543 format %{ "cmp $op1, $op2" %}
14544
14545 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14546
14547 ins_pipe(icmp_reg_imm);
14548 %}
14549
14550 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14551 %{
14552 match(Set cr (CmpP op1 op2));
14553
14554 effect(DEF cr, USE op1, USE op2);
14555
14556 ins_cost(INSN_COST);
14557 format %{ "cmp $op1, $op2\t // ptr" %}
14558
14559 ins_encode(aarch64_enc_cmpp(op1, op2));
14560
14561 ins_pipe(icmp_reg_reg);
14562 %}
14563
14564 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14565 %{
14566 match(Set cr (CmpN op1 op2));
14567
14568 effect(DEF cr, USE op1, USE op2);
14569
14570 ins_cost(INSN_COST);
14571 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14572
14573 ins_encode(aarch64_enc_cmpn(op1, op2));
14574
14575 ins_pipe(icmp_reg_reg);
14576 %}
14577
14578 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14579 %{
14580 match(Set cr (CmpP op1 zero));
14581
14582 effect(DEF cr, USE op1, USE zero);
14583
14584 ins_cost(INSN_COST);
14585 format %{ "cmp $op1, 0\t // ptr" %}
14586
14587 ins_encode(aarch64_enc_testp(op1));
14588
14589 ins_pipe(icmp_reg_imm);
14590 %}
14591
14592 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14593 %{
14594 match(Set cr (CmpN op1 zero));
14595
14596 effect(DEF cr, USE op1, USE zero);
14597
14598 ins_cost(INSN_COST);
14599 format %{ "cmp $op1, 0\t // compressed ptr" %}
14600
14601 ins_encode(aarch64_enc_testn(op1));
14602
14603 ins_pipe(icmp_reg_imm);
14604 %}
14605
14606 // FP comparisons
14607 //
14608 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14609 // using normal cmpOp. See declaration of rFlagsReg for details.
14610
14611 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14612 %{
14613 match(Set cr (CmpF src1 src2));
14614
14615 ins_cost(3 * INSN_COST);
14616 format %{ "fcmps $src1, $src2" %}
14617
14618 ins_encode %{
14619 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14620 %}
14621
14622 ins_pipe(pipe_class_compare);
14623 %}
14624
14625 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14626 %{
14627 match(Set cr (CmpF src1 src2));
14628
14629 ins_cost(3 * INSN_COST);
14630 format %{ "fcmps $src1, 0.0" %}
14631
14632 ins_encode %{
14633 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14634 %}
14635
14636 ins_pipe(pipe_class_compare);
14637 %}
14638 // FROM HERE
14639
14640 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14641 %{
14642 match(Set cr (CmpD src1 src2));
14643
14644 ins_cost(3 * INSN_COST);
14645 format %{ "fcmpd $src1, $src2" %}
14646
14647 ins_encode %{
14648 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14649 %}
14650
14651 ins_pipe(pipe_class_compare);
14652 %}
14653
14654 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14655 %{
14656 match(Set cr (CmpD src1 src2));
14657
14658 ins_cost(3 * INSN_COST);
14659 format %{ "fcmpd $src1, 0.0" %}
14660
14661 ins_encode %{
14662 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14663 %}
14664
14665 ins_pipe(pipe_class_compare);
14666 %}
14667
14668 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14669 %{
14670 match(Set dst (CmpF3 src1 src2));
14671 effect(KILL cr);
14672
14673 ins_cost(5 * INSN_COST);
14674 format %{ "fcmps $src1, $src2\n\t"
14675 "csinvw($dst, zr, zr, eq\n\t"
14676 "csnegw($dst, $dst, $dst, lt)"
14677 %}
14678
14679 ins_encode %{
14680 Label done;
14681 FloatRegister s1 = as_FloatRegister($src1$$reg);
14682 FloatRegister s2 = as_FloatRegister($src2$$reg);
14683 Register d = as_Register($dst$$reg);
14684 __ fcmps(s1, s2);
14685 // installs 0 if EQ else -1
14686 __ csinvw(d, zr, zr, Assembler::EQ);
14687 // keeps -1 if less or unordered else installs 1
14688 __ csnegw(d, d, d, Assembler::LT);
14689 __ bind(done);
14690 %}
14691
14692 ins_pipe(pipe_class_default);
14693
14694 %}
14695
14696 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14697 %{
14698 match(Set dst (CmpD3 src1 src2));
14699 effect(KILL cr);
14700
14701 ins_cost(5 * INSN_COST);
14702 format %{ "fcmpd $src1, $src2\n\t"
14703 "csinvw($dst, zr, zr, eq\n\t"
14704 "csnegw($dst, $dst, $dst, lt)"
14705 %}
14706
14707 ins_encode %{
14708 Label done;
14709 FloatRegister s1 = as_FloatRegister($src1$$reg);
14710 FloatRegister s2 = as_FloatRegister($src2$$reg);
14711 Register d = as_Register($dst$$reg);
14712 __ fcmpd(s1, s2);
14713 // installs 0 if EQ else -1
14714 __ csinvw(d, zr, zr, Assembler::EQ);
14715 // keeps -1 if less or unordered else installs 1
14716 __ csnegw(d, d, d, Assembler::LT);
14717 __ bind(done);
14718 %}
14719 ins_pipe(pipe_class_default);
14720
14721 %}
14722
14723 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14724 %{
14725 match(Set dst (CmpF3 src1 zero));
14726 effect(KILL cr);
14727
14728 ins_cost(5 * INSN_COST);
14729 format %{ "fcmps $src1, 0.0\n\t"
14730 "csinvw($dst, zr, zr, eq\n\t"
14731 "csnegw($dst, $dst, $dst, lt)"
14732 %}
14733
14734 ins_encode %{
14735 Label done;
14736 FloatRegister s1 = as_FloatRegister($src1$$reg);
14737 Register d = as_Register($dst$$reg);
14738 __ fcmps(s1, 0.0);
14739 // installs 0 if EQ else -1
14740 __ csinvw(d, zr, zr, Assembler::EQ);
14741 // keeps -1 if less or unordered else installs 1
14742 __ csnegw(d, d, d, Assembler::LT);
14743 __ bind(done);
14744 %}
14745
14746 ins_pipe(pipe_class_default);
14747
14748 %}
14749
14750 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14751 %{
14752 match(Set dst (CmpD3 src1 zero));
14753 effect(KILL cr);
14754
14755 ins_cost(5 * INSN_COST);
14756 format %{ "fcmpd $src1, 0.0\n\t"
14757 "csinvw($dst, zr, zr, eq\n\t"
14758 "csnegw($dst, $dst, $dst, lt)"
14759 %}
14760
14761 ins_encode %{
14762 Label done;
14763 FloatRegister s1 = as_FloatRegister($src1$$reg);
14764 Register d = as_Register($dst$$reg);
14765 __ fcmpd(s1, 0.0);
14766 // installs 0 if EQ else -1
14767 __ csinvw(d, zr, zr, Assembler::EQ);
14768 // keeps -1 if less or unordered else installs 1
14769 __ csnegw(d, d, d, Assembler::LT);
14770 __ bind(done);
14771 %}
14772 ins_pipe(pipe_class_default);
14773
14774 %}
14775
14776 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14777 %{
14778 match(Set dst (CmpLTMask p q));
14779 effect(KILL cr);
14780
14781 ins_cost(3 * INSN_COST);
14782
14783 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14784 "csetw $dst, lt\n\t"
14785 "subw $dst, zr, $dst"
14786 %}
14787
14788 ins_encode %{
14789 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14790 __ csetw(as_Register($dst$$reg), Assembler::LT);
14791 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14792 %}
14793
14794 ins_pipe(ialu_reg_reg);
14795 %}
14796
14797 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14798 %{
14799 match(Set dst (CmpLTMask src zero));
14800 effect(KILL cr);
14801
14802 ins_cost(INSN_COST);
14803
14804 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14805
14806 ins_encode %{
14807 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14808 %}
14809
14810 ins_pipe(ialu_reg_shift);
14811 %}
14812
14813 // ============================================================================
14814 // Max and Min
14815
14816 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14817
14818 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14819 %{
14820 effect(DEF cr, USE src);
14821 ins_cost(INSN_COST);
14822 format %{ "cmpw $src, 0" %}
14823
14824 ins_encode %{
14825 __ cmpw($src$$Register, 0);
14826 %}
14827 ins_pipe(icmp_reg_imm);
14828 %}
14829
14830 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14831 %{
14832 match(Set dst (MinI src1 src2));
14833 ins_cost(INSN_COST * 3);
14834
14835 expand %{
14836 rFlagsReg cr;
14837 compI_reg_reg(cr, src1, src2);
14838 cmovI_reg_reg_lt(dst, src1, src2, cr);
14839 %}
14840 %}
14841
14842 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14843 %{
14844 match(Set dst (MaxI src1 src2));
14845 ins_cost(INSN_COST * 3);
14846
14847 expand %{
14848 rFlagsReg cr;
14849 compI_reg_reg(cr, src1, src2);
14850 cmovI_reg_reg_gt(dst, src1, src2, cr);
14851 %}
14852 %}
14853
14854
14855 // ============================================================================
14856 // Branch Instructions
14857
14858 // Direct Branch.
14859 instruct branch(label lbl)
14860 %{
14861 match(Goto);
14862
14863 effect(USE lbl);
14864
14865 ins_cost(BRANCH_COST);
14866 format %{ "b $lbl" %}
14867
14868 ins_encode(aarch64_enc_b(lbl));
14869
14870 ins_pipe(pipe_branch);
14871 %}
14872
14873 // Conditional Near Branch
14874 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14875 %{
14876 // Same match rule as `branchConFar'.
14877 match(If cmp cr);
14878
14879 effect(USE lbl);
14880
14881 ins_cost(BRANCH_COST);
14882 // If set to 1 this indicates that the current instruction is a
14883 // short variant of a long branch. This avoids using this
14884 // instruction in first-pass matching. It will then only be used in
14885 // the `Shorten_branches' pass.
14886 // ins_short_branch(1);
14887 format %{ "b$cmp $lbl" %}
14888
14889 ins_encode(aarch64_enc_br_con(cmp, lbl));
14890
14891 ins_pipe(pipe_branch_cond);
14892 %}
14893
14894 // Conditional Near Branch Unsigned
14895 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14896 %{
14897 // Same match rule as `branchConFar'.
14898 match(If cmp cr);
14899
14900 effect(USE lbl);
14901
14902 ins_cost(BRANCH_COST);
14903 // If set to 1 this indicates that the current instruction is a
14904 // short variant of a long branch. This avoids using this
14905 // instruction in first-pass matching. It will then only be used in
14906 // the `Shorten_branches' pass.
14907 // ins_short_branch(1);
14908 format %{ "b$cmp $lbl\t# unsigned" %}
14909
14910 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14911
14912 ins_pipe(pipe_branch_cond);
14913 %}
14914
14915 // Make use of CBZ and CBNZ. These instructions, as well as being
14916 // shorter than (cmp; branch), have the additional benefit of not
14917 // killing the flags.
14918
14919 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14920 match(If cmp (CmpI op1 op2));
14921 effect(USE labl);
14922
14923 ins_cost(BRANCH_COST);
14924 format %{ "cbw$cmp $op1, $labl" %}
14925 ins_encode %{
14926 Label* L = $labl$$label;
14927 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14928 if (cond == Assembler::EQ)
14929 __ cbzw($op1$$Register, *L);
14930 else
14931 __ cbnzw($op1$$Register, *L);
14932 %}
14933 ins_pipe(pipe_cmp_branch);
14934 %}
14935
14936 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14937 match(If cmp (CmpL op1 op2));
14938 effect(USE labl);
14939
14940 ins_cost(BRANCH_COST);
14941 format %{ "cb$cmp $op1, $labl" %}
14942 ins_encode %{
14943 Label* L = $labl$$label;
14944 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14945 if (cond == Assembler::EQ)
14946 __ cbz($op1$$Register, *L);
14947 else
14948 __ cbnz($op1$$Register, *L);
14949 %}
14950 ins_pipe(pipe_cmp_branch);
14951 %}
14952
14953 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14954 match(If cmp (CmpP op1 op2));
14955 effect(USE labl);
14956
14957 ins_cost(BRANCH_COST);
14958 format %{ "cb$cmp $op1, $labl" %}
14959 ins_encode %{
14960 Label* L = $labl$$label;
14961 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14962 if (cond == Assembler::EQ)
14963 __ cbz($op1$$Register, *L);
14964 else
14965 __ cbnz($op1$$Register, *L);
14966 %}
14967 ins_pipe(pipe_cmp_branch);
14968 %}
14969
14970 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14971 match(If cmp (CmpN op1 op2));
14972 effect(USE labl);
14973
14974 ins_cost(BRANCH_COST);
14975 format %{ "cbw$cmp $op1, $labl" %}
14976 ins_encode %{
14977 Label* L = $labl$$label;
14978 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14979 if (cond == Assembler::EQ)
14980 __ cbzw($op1$$Register, *L);
14981 else
14982 __ cbnzw($op1$$Register, *L);
14983 %}
14984 ins_pipe(pipe_cmp_branch);
14985 %}
14986
14987 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
14988 match(If cmp (CmpP (DecodeN oop) zero));
14989 effect(USE labl);
14990
14991 ins_cost(BRANCH_COST);
14992 format %{ "cb$cmp $oop, $labl" %}
14993 ins_encode %{
14994 Label* L = $labl$$label;
14995 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14996 if (cond == Assembler::EQ)
14997 __ cbzw($oop$$Register, *L);
14998 else
14999 __ cbnzw($oop$$Register, *L);
15000 %}
15001 ins_pipe(pipe_cmp_branch);
15002 %}
15003
15004 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15005 match(If cmp (CmpU op1 op2));
15006 effect(USE labl);
15007
15008 ins_cost(BRANCH_COST);
15009 format %{ "cbw$cmp $op1, $labl" %}
15010 ins_encode %{
15011 Label* L = $labl$$label;
15012 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15013 if (cond == Assembler::EQ || cond == Assembler::LS) {
15014 __ cbzw($op1$$Register, *L);
15015 } else {
15016 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15017 __ cbnzw($op1$$Register, *L);
15018 }
15019 %}
15020 ins_pipe(pipe_cmp_branch);
15021 %}
15022
15023 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15024 match(If cmp (CmpUL op1 op2));
15025 effect(USE labl);
15026
15027 ins_cost(BRANCH_COST);
15028 format %{ "cb$cmp $op1, $labl" %}
15029 ins_encode %{
15030 Label* L = $labl$$label;
15031 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15032 if (cond == Assembler::EQ || cond == Assembler::LS) {
15033 __ cbz($op1$$Register, *L);
15034 } else {
15035 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15036 __ cbnz($op1$$Register, *L);
15037 }
15038 %}
15039 ins_pipe(pipe_cmp_branch);
15040 %}
15041
15042 // Test bit and Branch
15043
15044 // Patterns for short (< 32KiB) variants
15045 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15046 match(If cmp (CmpL op1 op2));
15047 effect(USE labl);
15048
15049 ins_cost(BRANCH_COST);
15050 format %{ "cb$cmp $op1, $labl # long" %}
15051 ins_encode %{
15052 Label* L = $labl$$label;
15053 Assembler::Condition cond =
15054 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15055 __ tbr(cond, $op1$$Register, 63, *L);
15056 %}
15057 ins_pipe(pipe_cmp_branch);
15058 ins_short_branch(1);
15059 %}
15060
15061 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15062 match(If cmp (CmpI op1 op2));
15063 effect(USE labl);
15064
15065 ins_cost(BRANCH_COST);
15066 format %{ "cb$cmp $op1, $labl # int" %}
15067 ins_encode %{
15068 Label* L = $labl$$label;
15069 Assembler::Condition cond =
15070 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15071 __ tbr(cond, $op1$$Register, 31, *L);
15072 %}
15073 ins_pipe(pipe_cmp_branch);
15074 ins_short_branch(1);
15075 %}
15076
15077 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15078 match(If cmp (CmpL (AndL op1 op2) op3));
15079 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15080 effect(USE labl);
15081
15082 ins_cost(BRANCH_COST);
15083 format %{ "tb$cmp $op1, $op2, $labl" %}
15084 ins_encode %{
15085 Label* L = $labl$$label;
15086 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15087 int bit = exact_log2_long($op2$$constant);
15088 __ tbr(cond, $op1$$Register, bit, *L);
15089 %}
15090 ins_pipe(pipe_cmp_branch);
15091 ins_short_branch(1);
15092 %}
15093
15094 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15095 match(If cmp (CmpI (AndI op1 op2) op3));
15096 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15097 effect(USE labl);
15098
15099 ins_cost(BRANCH_COST);
15100 format %{ "tb$cmp $op1, $op2, $labl" %}
15101 ins_encode %{
15102 Label* L = $labl$$label;
15103 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15104 int bit = exact_log2((juint)$op2$$constant);
15105 __ tbr(cond, $op1$$Register, bit, *L);
15106 %}
15107 ins_pipe(pipe_cmp_branch);
15108 ins_short_branch(1);
15109 %}
15110
15111 // And far variants
15112 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15113 match(If cmp (CmpL op1 op2));
15114 effect(USE labl);
15115
15116 ins_cost(BRANCH_COST);
15117 format %{ "cb$cmp $op1, $labl # long" %}
15118 ins_encode %{
15119 Label* L = $labl$$label;
15120 Assembler::Condition cond =
15121 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15122 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15123 %}
15124 ins_pipe(pipe_cmp_branch);
15125 %}
15126
15127 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15128 match(If cmp (CmpI op1 op2));
15129 effect(USE labl);
15130
15131 ins_cost(BRANCH_COST);
15132 format %{ "cb$cmp $op1, $labl # int" %}
15133 ins_encode %{
15134 Label* L = $labl$$label;
15135 Assembler::Condition cond =
15136 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15137 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15138 %}
15139 ins_pipe(pipe_cmp_branch);
15140 %}
15141
15142 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15143 match(If cmp (CmpL (AndL op1 op2) op3));
15144 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15145 effect(USE labl);
15146
15147 ins_cost(BRANCH_COST);
15148 format %{ "tb$cmp $op1, $op2, $labl" %}
15149 ins_encode %{
15150 Label* L = $labl$$label;
15151 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15152 int bit = exact_log2_long($op2$$constant);
15153 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15154 %}
15155 ins_pipe(pipe_cmp_branch);
15156 %}
15157
15158 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15159 match(If cmp (CmpI (AndI op1 op2) op3));
15160 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15161 effect(USE labl);
15162
15163 ins_cost(BRANCH_COST);
15164 format %{ "tb$cmp $op1, $op2, $labl" %}
15165 ins_encode %{
15166 Label* L = $labl$$label;
15167 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15168 int bit = exact_log2((juint)$op2$$constant);
15169 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15170 %}
15171 ins_pipe(pipe_cmp_branch);
15172 %}
15173
15174 // Test bits
15175
15176 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15177 match(Set cr (CmpL (AndL op1 op2) op3));
15178 predicate(Assembler::operand_valid_for_logical_immediate
15179 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15180
15181 ins_cost(INSN_COST);
15182 format %{ "tst $op1, $op2 # long" %}
15183 ins_encode %{
15184 __ tst($op1$$Register, $op2$$constant);
15185 %}
15186 ins_pipe(ialu_reg_reg);
15187 %}
15188
15189 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15190 match(Set cr (CmpI (AndI op1 op2) op3));
15191 predicate(Assembler::operand_valid_for_logical_immediate
15192 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15193
15194 ins_cost(INSN_COST);
15195 format %{ "tst $op1, $op2 # int" %}
15196 ins_encode %{
15197 __ tstw($op1$$Register, $op2$$constant);
15198 %}
15199 ins_pipe(ialu_reg_reg);
15200 %}
15201
15202 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15203 match(Set cr (CmpL (AndL op1 op2) op3));
15204
15205 ins_cost(INSN_COST);
15206 format %{ "tst $op1, $op2 # long" %}
15207 ins_encode %{
15208 __ tst($op1$$Register, $op2$$Register);
15209 %}
15210 ins_pipe(ialu_reg_reg);
15211 %}
15212
15213 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15214 match(Set cr (CmpI (AndI op1 op2) op3));
15215
15216 ins_cost(INSN_COST);
15217 format %{ "tstw $op1, $op2 # int" %}
15218 ins_encode %{
15219 __ tstw($op1$$Register, $op2$$Register);
15220 %}
15221 ins_pipe(ialu_reg_reg);
15222 %}
15223
15224
15225 // Conditional Far Branch
15226 // Conditional Far Branch Unsigned
15227 // TODO: fixme
15228
15229 // counted loop end branch near
15230 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15231 %{
15232 match(CountedLoopEnd cmp cr);
15233
15234 effect(USE lbl);
15235
15236 ins_cost(BRANCH_COST);
15237 // short variant.
15238 // ins_short_branch(1);
15239 format %{ "b$cmp $lbl \t// counted loop end" %}
15240
15241 ins_encode(aarch64_enc_br_con(cmp, lbl));
15242
15243 ins_pipe(pipe_branch);
15244 %}
15245
15246 // counted loop end branch far
15247 // TODO: fixme
15248
15249 // ============================================================================
15250 // inlined locking and unlocking
15251
15252 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15253 %{
15254 match(Set cr (FastLock object box));
15255 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15256
15257 ins_cost(5 * INSN_COST);
15258 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15259
15260 ins_encode %{
15261 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15262 %}
15263
15264 ins_pipe(pipe_serial);
15265 %}
15266
15267 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15268 %{
15269 match(Set cr (FastUnlock object box));
15270 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15271
15272 ins_cost(5 * INSN_COST);
15273 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15274
15275 ins_encode %{
15276 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15277 %}
15278
15279 ins_pipe(pipe_serial);
15280 %}
15281
15282 // ============================================================================
15283 // Safepoint Instructions
15284
15285 // TODO
15286 // provide a near and far version of this code
15287
15288 instruct safePoint(rFlagsReg cr, iRegP poll)
15289 %{
15290 match(SafePoint poll);
15291 effect(KILL cr);
15292
15293 format %{
15294 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15295 %}
15296 ins_encode %{
15297 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15298 %}
15299 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15300 %}
15301
15302
15303 // ============================================================================
15304 // Procedure Call/Return Instructions
15305
15306 // Call Java Static Instruction
15307
15308 instruct CallStaticJavaDirect(method meth)
15309 %{
15310 match(CallStaticJava);
15311
15312 effect(USE meth);
15313
15314 ins_cost(CALL_COST);
15315
15316 format %{ "call,static $meth \t// ==> " %}
15317
15318 ins_encode(aarch64_enc_java_static_call(meth),
15319 aarch64_enc_call_epilog);
15320
15321 ins_pipe(pipe_class_call);
15322 %}
15323
15324 // TO HERE
15325
15326 // Call Java Dynamic Instruction
15327 instruct CallDynamicJavaDirect(method meth)
15328 %{
15329 match(CallDynamicJava);
15330
15331 effect(USE meth);
15332
15333 ins_cost(CALL_COST);
15334
15335 format %{ "CALL,dynamic $meth \t// ==> " %}
15336
15337 ins_encode(aarch64_enc_java_dynamic_call(meth),
15338 aarch64_enc_call_epilog);
15339
15340 ins_pipe(pipe_class_call);
15341 %}
15342
15343 // Call Runtime Instruction
15344
15345 instruct CallRuntimeDirect(method meth)
15346 %{
15347 match(CallRuntime);
15348
15349 effect(USE meth);
15350
15351 ins_cost(CALL_COST);
15352
15353 format %{ "CALL, runtime $meth" %}
15354
15355 ins_encode( aarch64_enc_java_to_runtime(meth) );
15356
15357 ins_pipe(pipe_class_call);
15358 %}
15359
15360 // Call Runtime Instruction
15361
15362 instruct CallLeafDirect(method meth)
15363 %{
15364 match(CallLeaf);
15365
15366 effect(USE meth);
15367
15368 ins_cost(CALL_COST);
15369
15370 format %{ "CALL, runtime leaf $meth" %}
15371
15372 ins_encode( aarch64_enc_java_to_runtime(meth) );
15373
15374 ins_pipe(pipe_class_call);
15375 %}
15376
15377 // Call Runtime Instruction without safepoint and with vector arguments
15378 instruct CallLeafDirectVector(method meth)
15379 %{
15380 match(CallLeafVector);
15381
15382 effect(USE meth);
15383
15384 ins_cost(CALL_COST);
15385
15386 format %{ "CALL, runtime leaf vector $meth" %}
15387
15388 ins_encode(aarch64_enc_java_to_runtime(meth));
15389
15390 ins_pipe(pipe_class_call);
15391 %}
15392
15393 // Call Runtime Instruction
15394
15395 instruct CallLeafNoFPDirect(method meth)
15396 %{
15397 match(CallLeafNoFP);
15398
15399 effect(USE meth);
15400
15401 ins_cost(CALL_COST);
15402
15403 format %{ "CALL, runtime leaf nofp $meth" %}
15404
15405 ins_encode( aarch64_enc_java_to_runtime(meth) );
15406
15407 ins_pipe(pipe_class_call);
15408 %}
15409
15410 // Tail Call; Jump from runtime stub to Java code.
15411 // Also known as an 'interprocedural jump'.
15412 // Target of jump will eventually return to caller.
15413 // TailJump below removes the return address.
15414 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15415 // emitted just above the TailCall which has reset rfp to the caller state.
15416 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15417 %{
15418 match(TailCall jump_target method_ptr);
15419
15420 ins_cost(CALL_COST);
15421
15422 format %{ "br $jump_target\t# $method_ptr holds method" %}
15423
15424 ins_encode(aarch64_enc_tail_call(jump_target));
15425
15426 ins_pipe(pipe_class_call);
15427 %}
15428
15429 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15430 %{
15431 match(TailJump jump_target ex_oop);
15432
15433 ins_cost(CALL_COST);
15434
15435 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15436
15437 ins_encode(aarch64_enc_tail_jmp(jump_target));
15438
15439 ins_pipe(pipe_class_call);
15440 %}
15441
15442 // Forward exception.
15443 instruct ForwardExceptionjmp()
15444 %{
15445 match(ForwardException);
15446 ins_cost(CALL_COST);
15447
15448 format %{ "b forward_exception_stub" %}
15449 ins_encode %{
15450 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15451 %}
15452 ins_pipe(pipe_class_call);
15453 %}
15454
15455 // Create exception oop: created by stack-crawling runtime code.
15456 // Created exception is now available to this handler, and is setup
15457 // just prior to jumping to this handler. No code emitted.
15458 // TODO check
15459 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15460 instruct CreateException(iRegP_R0 ex_oop)
15461 %{
15462 match(Set ex_oop (CreateEx));
15463
15464 format %{ " -- \t// exception oop; no code emitted" %}
15465
15466 size(0);
15467
15468 ins_encode( /*empty*/ );
15469
15470 ins_pipe(pipe_class_empty);
15471 %}
15472
15473 // Rethrow exception: The exception oop will come in the first
15474 // argument position. Then JUMP (not call) to the rethrow stub code.
15475 instruct RethrowException() %{
15476 match(Rethrow);
15477 ins_cost(CALL_COST);
15478
15479 format %{ "b rethrow_stub" %}
15480
15481 ins_encode( aarch64_enc_rethrow() );
15482
15483 ins_pipe(pipe_class_call);
15484 %}
15485
15486
15487 // Return Instruction
15488 // epilog node loads ret address into lr as part of frame pop
15489 instruct Ret()
15490 %{
15491 match(Return);
15492
15493 format %{ "ret\t// return register" %}
15494
15495 ins_encode( aarch64_enc_ret() );
15496
15497 ins_pipe(pipe_branch);
15498 %}
15499
15500 // Die now.
15501 instruct ShouldNotReachHere() %{
15502 match(Halt);
15503
15504 ins_cost(CALL_COST);
15505 format %{ "ShouldNotReachHere" %}
15506
15507 ins_encode %{
15508 if (is_reachable()) {
15509 const char* str = __ code_string(_halt_reason);
15510 __ stop(str);
15511 }
15512 %}
15513
15514 ins_pipe(pipe_class_default);
15515 %}
15516
15517 // ============================================================================
15518 // Partial Subtype Check
15519 //
15520 // superklass array for an instance of the superklass. Set a hidden
15521 // internal cache on a hit (cache is checked with exposed code in
15522 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15523 // encoding ALSO sets flags.
15524
15525 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15526 %{
15527 match(Set result (PartialSubtypeCheck sub super));
15528 predicate(!UseSecondarySupersTable);
15529 effect(KILL cr, KILL temp);
15530
15531 ins_cost(20 * INSN_COST); // slightly larger than the next version
15532 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15533
15534 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15535
15536 opcode(0x1); // Force zero of result reg on hit
15537
15538 ins_pipe(pipe_class_memory);
15539 %}
15540
15541 // Two versions of partialSubtypeCheck, both used when we need to
15542 // search for a super class in the secondary supers array. The first
15543 // is used when we don't know _a priori_ the class being searched
15544 // for. The second, far more common, is used when we do know: this is
15545 // used for instanceof, checkcast, and any case where C2 can determine
15546 // it by constant propagation.
15547
15548 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15549 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15550 rFlagsReg cr)
15551 %{
15552 match(Set result (PartialSubtypeCheck sub super));
15553 predicate(UseSecondarySupersTable);
15554 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15555
15556 ins_cost(10 * INSN_COST); // slightly larger than the next version
15557 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15558
15559 ins_encode %{
15560 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15561 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15562 $vtemp$$FloatRegister,
15563 $result$$Register, /*L_success*/nullptr);
15564 %}
15565
15566 ins_pipe(pipe_class_memory);
15567 %}
15568
15569 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15570 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15571 rFlagsReg cr)
15572 %{
15573 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15574 predicate(UseSecondarySupersTable);
15575 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15576
15577 ins_cost(5 * INSN_COST); // smaller than the next version
15578 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15579
15580 ins_encode %{
15581 bool success = false;
15582 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15583 if (InlineSecondarySupersTest) {
15584 success =
15585 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15586 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15587 $vtemp$$FloatRegister,
15588 $result$$Register,
15589 super_klass_slot);
15590 } else {
15591 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15592 success = (call != nullptr);
15593 }
15594 if (!success) {
15595 ciEnv::current()->record_failure("CodeCache is full");
15596 return;
15597 }
15598 %}
15599
15600 ins_pipe(pipe_class_memory);
15601 %}
15602
15603 // Intrisics for String.compareTo()
15604
15605 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15606 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15607 %{
15608 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15609 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15610 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15611
15612 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15613 ins_encode %{
15614 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15615 __ string_compare($str1$$Register, $str2$$Register,
15616 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15617 $tmp1$$Register, $tmp2$$Register,
15618 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15619 %}
15620 ins_pipe(pipe_class_memory);
15621 %}
15622
15623 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15624 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15625 %{
15626 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15627 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15628 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15629
15630 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15631 ins_encode %{
15632 __ string_compare($str1$$Register, $str2$$Register,
15633 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15634 $tmp1$$Register, $tmp2$$Register,
15635 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15636 %}
15637 ins_pipe(pipe_class_memory);
15638 %}
15639
15640 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15641 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15642 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15643 %{
15644 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15645 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15646 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15647 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15648
15649 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15650 ins_encode %{
15651 __ string_compare($str1$$Register, $str2$$Register,
15652 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15653 $tmp1$$Register, $tmp2$$Register,
15654 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15655 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15656 %}
15657 ins_pipe(pipe_class_memory);
15658 %}
15659
15660 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15661 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15662 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15663 %{
15664 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15665 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15666 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15667 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15668
15669 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15670 ins_encode %{
15671 __ string_compare($str1$$Register, $str2$$Register,
15672 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15673 $tmp1$$Register, $tmp2$$Register,
15674 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15675 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15676 %}
15677 ins_pipe(pipe_class_memory);
15678 %}
15679
15680 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15681 // these string_compare variants as NEON register type for convenience so that the prototype of
15682 // string_compare can be shared with all variants.
15683
15684 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15685 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15686 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15687 pRegGov_P1 pgtmp2, rFlagsReg cr)
15688 %{
15689 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15690 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15691 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15692 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15693
15694 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15695 ins_encode %{
15696 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15697 __ string_compare($str1$$Register, $str2$$Register,
15698 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15699 $tmp1$$Register, $tmp2$$Register,
15700 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15701 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15702 StrIntrinsicNode::LL);
15703 %}
15704 ins_pipe(pipe_class_memory);
15705 %}
15706
15707 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15708 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15709 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15710 pRegGov_P1 pgtmp2, rFlagsReg cr)
15711 %{
15712 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15713 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15714 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15715 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15716
15717 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15718 ins_encode %{
15719 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15720 __ string_compare($str1$$Register, $str2$$Register,
15721 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15722 $tmp1$$Register, $tmp2$$Register,
15723 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15724 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15725 StrIntrinsicNode::LU);
15726 %}
15727 ins_pipe(pipe_class_memory);
15728 %}
15729
15730 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15731 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15732 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15733 pRegGov_P1 pgtmp2, rFlagsReg cr)
15734 %{
15735 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15736 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15737 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15738 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15739
15740 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15741 ins_encode %{
15742 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15743 __ string_compare($str1$$Register, $str2$$Register,
15744 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15745 $tmp1$$Register, $tmp2$$Register,
15746 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15747 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15748 StrIntrinsicNode::UL);
15749 %}
15750 ins_pipe(pipe_class_memory);
15751 %}
15752
15753 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15754 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15755 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15756 pRegGov_P1 pgtmp2, rFlagsReg cr)
15757 %{
15758 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15759 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15760 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15761 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15762
15763 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15764 ins_encode %{
15765 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15766 __ string_compare($str1$$Register, $str2$$Register,
15767 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15768 $tmp1$$Register, $tmp2$$Register,
15769 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15770 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15771 StrIntrinsicNode::UU);
15772 %}
15773 ins_pipe(pipe_class_memory);
15774 %}
15775
15776 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15777 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15778 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15779 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15780 %{
15781 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15782 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15783 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15784 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15785 TEMP vtmp0, TEMP vtmp1, KILL cr);
15786 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15787 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15788
15789 ins_encode %{
15790 __ string_indexof($str1$$Register, $str2$$Register,
15791 $cnt1$$Register, $cnt2$$Register,
15792 $tmp1$$Register, $tmp2$$Register,
15793 $tmp3$$Register, $tmp4$$Register,
15794 $tmp5$$Register, $tmp6$$Register,
15795 -1, $result$$Register, StrIntrinsicNode::UU);
15796 %}
15797 ins_pipe(pipe_class_memory);
15798 %}
15799
15800 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15801 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15802 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15803 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15804 %{
15805 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15806 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15807 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15808 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15809 TEMP vtmp0, TEMP vtmp1, KILL cr);
15810 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15811 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15812
15813 ins_encode %{
15814 __ string_indexof($str1$$Register, $str2$$Register,
15815 $cnt1$$Register, $cnt2$$Register,
15816 $tmp1$$Register, $tmp2$$Register,
15817 $tmp3$$Register, $tmp4$$Register,
15818 $tmp5$$Register, $tmp6$$Register,
15819 -1, $result$$Register, StrIntrinsicNode::LL);
15820 %}
15821 ins_pipe(pipe_class_memory);
15822 %}
15823
15824 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15825 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15826 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15827 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15828 %{
15829 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15830 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15831 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15832 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15833 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15834 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15835 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15836
15837 ins_encode %{
15838 __ string_indexof($str1$$Register, $str2$$Register,
15839 $cnt1$$Register, $cnt2$$Register,
15840 $tmp1$$Register, $tmp2$$Register,
15841 $tmp3$$Register, $tmp4$$Register,
15842 $tmp5$$Register, $tmp6$$Register,
15843 -1, $result$$Register, StrIntrinsicNode::UL);
15844 %}
15845 ins_pipe(pipe_class_memory);
15846 %}
15847
15848 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15849 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15850 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15851 %{
15852 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15853 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15854 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15855 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15856 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15857 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15858
15859 ins_encode %{
15860 int icnt2 = (int)$int_cnt2$$constant;
15861 __ string_indexof($str1$$Register, $str2$$Register,
15862 $cnt1$$Register, zr,
15863 $tmp1$$Register, $tmp2$$Register,
15864 $tmp3$$Register, $tmp4$$Register, zr, zr,
15865 icnt2, $result$$Register, StrIntrinsicNode::UU);
15866 %}
15867 ins_pipe(pipe_class_memory);
15868 %}
15869
15870 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15871 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15872 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15873 %{
15874 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15875 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15876 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15877 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15878 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15879 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15880
15881 ins_encode %{
15882 int icnt2 = (int)$int_cnt2$$constant;
15883 __ string_indexof($str1$$Register, $str2$$Register,
15884 $cnt1$$Register, zr,
15885 $tmp1$$Register, $tmp2$$Register,
15886 $tmp3$$Register, $tmp4$$Register, zr, zr,
15887 icnt2, $result$$Register, StrIntrinsicNode::LL);
15888 %}
15889 ins_pipe(pipe_class_memory);
15890 %}
15891
15892 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15893 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15894 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15895 %{
15896 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15897 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15898 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15899 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15900 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15901 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15902
15903 ins_encode %{
15904 int icnt2 = (int)$int_cnt2$$constant;
15905 __ string_indexof($str1$$Register, $str2$$Register,
15906 $cnt1$$Register, zr,
15907 $tmp1$$Register, $tmp2$$Register,
15908 $tmp3$$Register, $tmp4$$Register, zr, zr,
15909 icnt2, $result$$Register, StrIntrinsicNode::UL);
15910 %}
15911 ins_pipe(pipe_class_memory);
15912 %}
15913
15914 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15915 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15916 iRegINoSp tmp3, rFlagsReg cr)
15917 %{
15918 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15919 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15920 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15921 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15922
15923 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15924
15925 ins_encode %{
15926 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15927 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15928 $tmp3$$Register);
15929 %}
15930 ins_pipe(pipe_class_memory);
15931 %}
15932
15933 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15934 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15935 iRegINoSp tmp3, rFlagsReg cr)
15936 %{
15937 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15938 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15939 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15940 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15941
15942 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15943
15944 ins_encode %{
15945 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15946 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15947 $tmp3$$Register);
15948 %}
15949 ins_pipe(pipe_class_memory);
15950 %}
15951
15952 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15953 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15954 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15955 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15956 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15957 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15958 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15959 ins_encode %{
15960 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15961 $result$$Register, $ztmp1$$FloatRegister,
15962 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15963 $ptmp$$PRegister, true /* isL */);
15964 %}
15965 ins_pipe(pipe_class_memory);
15966 %}
15967
15968 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15969 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15970 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15971 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
15972 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15973 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15974 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15975 ins_encode %{
15976 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15977 $result$$Register, $ztmp1$$FloatRegister,
15978 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15979 $ptmp$$PRegister, false /* isL */);
15980 %}
15981 ins_pipe(pipe_class_memory);
15982 %}
15983
15984 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15985 iRegI_R0 result, rFlagsReg cr)
15986 %{
15987 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
15988 match(Set result (StrEquals (Binary str1 str2) cnt));
15989 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15990
15991 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15992 ins_encode %{
15993 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15994 __ string_equals($str1$$Register, $str2$$Register,
15995 $result$$Register, $cnt$$Register);
15996 %}
15997 ins_pipe(pipe_class_memory);
15998 %}
15999
16000 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16001 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16002 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16003 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16004 iRegP_R10 tmp, rFlagsReg cr)
16005 %{
16006 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16007 match(Set result (AryEq ary1 ary2));
16008 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16009 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16010 TEMP vtmp6, TEMP vtmp7, KILL cr);
16011
16012 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16013 ins_encode %{
16014 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16015 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16016 $result$$Register, $tmp$$Register, 1);
16017 if (tpc == nullptr) {
16018 ciEnv::current()->record_failure("CodeCache is full");
16019 return;
16020 }
16021 %}
16022 ins_pipe(pipe_class_memory);
16023 %}
16024
16025 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16026 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16027 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16028 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16029 iRegP_R10 tmp, rFlagsReg cr)
16030 %{
16031 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16032 match(Set result (AryEq ary1 ary2));
16033 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16034 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16035 TEMP vtmp6, TEMP vtmp7, KILL cr);
16036
16037 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16038 ins_encode %{
16039 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16040 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16041 $result$$Register, $tmp$$Register, 2);
16042 if (tpc == nullptr) {
16043 ciEnv::current()->record_failure("CodeCache is full");
16044 return;
16045 }
16046 %}
16047 ins_pipe(pipe_class_memory);
16048 %}
16049
16050 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16051 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16052 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16053 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16054 %{
16055 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16056 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16057 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16058
16059 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16060 ins_encode %{
16061 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16062 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16063 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16064 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16065 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16066 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16067 (BasicType)$basic_type$$constant);
16068 if (tpc == nullptr) {
16069 ciEnv::current()->record_failure("CodeCache is full");
16070 return;
16071 }
16072 %}
16073 ins_pipe(pipe_class_memory);
16074 %}
16075
16076 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16077 %{
16078 match(Set result (CountPositives ary1 len));
16079 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16080 format %{ "count positives byte[] $ary1,$len -> $result" %}
16081 ins_encode %{
16082 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16083 if (tpc == nullptr) {
16084 ciEnv::current()->record_failure("CodeCache is full");
16085 return;
16086 }
16087 %}
16088 ins_pipe( pipe_slow );
16089 %}
16090
16091 // fast char[] to byte[] compression
16092 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16093 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16094 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16095 iRegI_R0 result, rFlagsReg cr)
16096 %{
16097 match(Set result (StrCompressedCopy src (Binary dst len)));
16098 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16099 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16100
16101 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16102 ins_encode %{
16103 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16104 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16105 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16106 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16107 %}
16108 ins_pipe(pipe_slow);
16109 %}
16110
16111 // fast byte[] to char[] inflation
16112 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16113 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16114 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16115 %{
16116 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16117 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16118 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16119 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16120
16121 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16122 ins_encode %{
16123 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16124 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16125 $vtmp2$$FloatRegister, $tmp$$Register);
16126 if (tpc == nullptr) {
16127 ciEnv::current()->record_failure("CodeCache is full");
16128 return;
16129 }
16130 %}
16131 ins_pipe(pipe_class_memory);
16132 %}
16133
16134 // encode char[] to byte[] in ISO_8859_1
16135 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16136 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16137 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16138 iRegI_R0 result, rFlagsReg cr)
16139 %{
16140 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16141 match(Set result (EncodeISOArray src (Binary dst len)));
16142 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16143 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16144
16145 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16146 ins_encode %{
16147 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16148 $result$$Register, false,
16149 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16150 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16151 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16152 %}
16153 ins_pipe(pipe_class_memory);
16154 %}
16155
16156 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16157 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16158 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16159 iRegI_R0 result, rFlagsReg cr)
16160 %{
16161 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16162 match(Set result (EncodeISOArray src (Binary dst len)));
16163 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16164 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16165
16166 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16167 ins_encode %{
16168 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16169 $result$$Register, true,
16170 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16171 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16172 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16173 %}
16174 ins_pipe(pipe_class_memory);
16175 %}
16176
16177 //----------------------------- CompressBits/ExpandBits ------------------------
16178
16179 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16180 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16181 match(Set dst (CompressBits src mask));
16182 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16183 format %{ "mov $tsrc, $src\n\t"
16184 "mov $tmask, $mask\n\t"
16185 "bext $tdst, $tsrc, $tmask\n\t"
16186 "mov $dst, $tdst"
16187 %}
16188 ins_encode %{
16189 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16190 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16191 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16192 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16193 %}
16194 ins_pipe(pipe_slow);
16195 %}
16196
16197 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16198 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16199 match(Set dst (CompressBits (LoadI mem) mask));
16200 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16201 format %{ "ldrs $tsrc, $mem\n\t"
16202 "ldrs $tmask, $mask\n\t"
16203 "bext $tdst, $tsrc, $tmask\n\t"
16204 "mov $dst, $tdst"
16205 %}
16206 ins_encode %{
16207 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16208 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16209 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16210 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16211 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16212 %}
16213 ins_pipe(pipe_slow);
16214 %}
16215
16216 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16217 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16218 match(Set dst (CompressBits src mask));
16219 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16220 format %{ "mov $tsrc, $src\n\t"
16221 "mov $tmask, $mask\n\t"
16222 "bext $tdst, $tsrc, $tmask\n\t"
16223 "mov $dst, $tdst"
16224 %}
16225 ins_encode %{
16226 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16227 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16228 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16229 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16230 %}
16231 ins_pipe(pipe_slow);
16232 %}
16233
16234 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16235 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16236 match(Set dst (CompressBits (LoadL mem) mask));
16237 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16238 format %{ "ldrd $tsrc, $mem\n\t"
16239 "ldrd $tmask, $mask\n\t"
16240 "bext $tdst, $tsrc, $tmask\n\t"
16241 "mov $dst, $tdst"
16242 %}
16243 ins_encode %{
16244 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16245 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16246 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16247 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16248 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16249 %}
16250 ins_pipe(pipe_slow);
16251 %}
16252
16253 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16254 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16255 match(Set dst (ExpandBits src mask));
16256 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16257 format %{ "mov $tsrc, $src\n\t"
16258 "mov $tmask, $mask\n\t"
16259 "bdep $tdst, $tsrc, $tmask\n\t"
16260 "mov $dst, $tdst"
16261 %}
16262 ins_encode %{
16263 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16264 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16265 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16266 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16267 %}
16268 ins_pipe(pipe_slow);
16269 %}
16270
16271 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16272 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16273 match(Set dst (ExpandBits (LoadI mem) mask));
16274 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16275 format %{ "ldrs $tsrc, $mem\n\t"
16276 "ldrs $tmask, $mask\n\t"
16277 "bdep $tdst, $tsrc, $tmask\n\t"
16278 "mov $dst, $tdst"
16279 %}
16280 ins_encode %{
16281 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16282 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16283 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16284 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16285 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16286 %}
16287 ins_pipe(pipe_slow);
16288 %}
16289
16290 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16291 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16292 match(Set dst (ExpandBits src mask));
16293 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16294 format %{ "mov $tsrc, $src\n\t"
16295 "mov $tmask, $mask\n\t"
16296 "bdep $tdst, $tsrc, $tmask\n\t"
16297 "mov $dst, $tdst"
16298 %}
16299 ins_encode %{
16300 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16301 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16302 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16303 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16304 %}
16305 ins_pipe(pipe_slow);
16306 %}
16307
16308
16309 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16310 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16311 match(Set dst (ExpandBits (LoadL mem) mask));
16312 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16313 format %{ "ldrd $tsrc, $mem\n\t"
16314 "ldrd $tmask, $mask\n\t"
16315 "bdep $tdst, $tsrc, $tmask\n\t"
16316 "mov $dst, $tdst"
16317 %}
16318 ins_encode %{
16319 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16320 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16321 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16322 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16323 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16324 %}
16325 ins_pipe(pipe_slow);
16326 %}
16327
16328 //----------------------------- Reinterpret ----------------------------------
16329 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16330 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16331 match(Set dst (ReinterpretHF2S src));
16332 format %{ "reinterpretHF2S $dst, $src" %}
16333 ins_encode %{
16334 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16335 %}
16336 ins_pipe(pipe_slow);
16337 %}
16338
16339 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16340 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16341 match(Set dst (ReinterpretS2HF src));
16342 format %{ "reinterpretS2HF $dst, $src" %}
16343 ins_encode %{
16344 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16345 %}
16346 ins_pipe(pipe_slow);
16347 %}
16348
16349 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16350 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16351 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16352 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16353 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16354 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16355 // can be omitted in this pattern, resulting in -
16356 // fcvt $dst, $src // Convert float to half-precision float
16357 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16358 %{
16359 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16360 format %{ "convF2HFAndS2HF $dst, $src" %}
16361 ins_encode %{
16362 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16363 %}
16364 ins_pipe(pipe_slow);
16365 %}
16366
16367 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16368 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16369 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16370 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16371 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16372 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16373 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16374 // resulting in -
16375 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16376 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16377 %{
16378 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16379 format %{ "convHF2SAndHF2F $dst, $src" %}
16380 ins_encode %{
16381 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16382 %}
16383 ins_pipe(pipe_slow);
16384 %}
16385
16386 // ============================================================================
16387 // This name is KNOWN by the ADLC and cannot be changed.
16388 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16389 // for this guy.
16390 instruct tlsLoadP(thread_RegP dst)
16391 %{
16392 match(Set dst (ThreadLocal));
16393
16394 ins_cost(0);
16395
16396 format %{ " -- \t// $dst=Thread::current(), empty" %}
16397
16398 size(0);
16399
16400 ins_encode( /*empty*/ );
16401
16402 ins_pipe(pipe_class_empty);
16403 %}
16404
16405 //----------PEEPHOLE RULES-----------------------------------------------------
16406 // These must follow all instruction definitions as they use the names
16407 // defined in the instructions definitions.
16408 //
16409 // peepmatch ( root_instr_name [preceding_instruction]* );
16410 //
16411 // peepconstraint %{
16412 // (instruction_number.operand_name relational_op instruction_number.operand_name
16413 // [, ...] );
16414 // // instruction numbers are zero-based using left to right order in peepmatch
16415 //
16416 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16417 // // provide an instruction_number.operand_name for each operand that appears
16418 // // in the replacement instruction's match rule
16419 //
16420 // ---------VM FLAGS---------------------------------------------------------
16421 //
16422 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16423 //
16424 // Each peephole rule is given an identifying number starting with zero and
16425 // increasing by one in the order seen by the parser. An individual peephole
16426 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16427 // on the command-line.
16428 //
16429 // ---------CURRENT LIMITATIONS----------------------------------------------
16430 //
16431 // Only match adjacent instructions in same basic block
16432 // Only equality constraints
16433 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16434 // Only one replacement instruction
16435 //
16436 // ---------EXAMPLE----------------------------------------------------------
16437 //
16438 // // pertinent parts of existing instructions in architecture description
16439 // instruct movI(iRegINoSp dst, iRegI src)
16440 // %{
16441 // match(Set dst (CopyI src));
16442 // %}
16443 //
16444 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16445 // %{
16446 // match(Set dst (AddI dst src));
16447 // effect(KILL cr);
16448 // %}
16449 //
16450 // // Change (inc mov) to lea
16451 // peephole %{
16452 // // increment preceded by register-register move
16453 // peepmatch ( incI_iReg movI );
16454 // // require that the destination register of the increment
16455 // // match the destination register of the move
16456 // peepconstraint ( 0.dst == 1.dst );
16457 // // construct a replacement instruction that sets
16458 // // the destination to ( move's source register + one )
16459 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16460 // %}
16461 //
16462
16463 // Implementation no longer uses movX instructions since
16464 // machine-independent system no longer uses CopyX nodes.
16465 //
16466 // peephole
16467 // %{
16468 // peepmatch (incI_iReg movI);
16469 // peepconstraint (0.dst == 1.dst);
16470 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16471 // %}
16472
16473 // peephole
16474 // %{
16475 // peepmatch (decI_iReg movI);
16476 // peepconstraint (0.dst == 1.dst);
16477 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16478 // %}
16479
16480 // peephole
16481 // %{
16482 // peepmatch (addI_iReg_imm movI);
16483 // peepconstraint (0.dst == 1.dst);
16484 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16485 // %}
16486
16487 // peephole
16488 // %{
16489 // peepmatch (incL_iReg movL);
16490 // peepconstraint (0.dst == 1.dst);
16491 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16492 // %}
16493
16494 // peephole
16495 // %{
16496 // peepmatch (decL_iReg movL);
16497 // peepconstraint (0.dst == 1.dst);
16498 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16499 // %}
16500
16501 // peephole
16502 // %{
16503 // peepmatch (addL_iReg_imm movL);
16504 // peepconstraint (0.dst == 1.dst);
16505 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16506 // %}
16507
16508 // peephole
16509 // %{
16510 // peepmatch (addP_iReg_imm movP);
16511 // peepconstraint (0.dst == 1.dst);
16512 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16513 // %}
16514
16515 // // Change load of spilled value to only a spill
16516 // instruct storeI(memory mem, iRegI src)
16517 // %{
16518 // match(Set mem (StoreI mem src));
16519 // %}
16520 //
16521 // instruct loadI(iRegINoSp dst, memory mem)
16522 // %{
16523 // match(Set dst (LoadI mem));
16524 // %}
16525 //
16526
16527 //----------SMARTSPILL RULES---------------------------------------------------
16528 // These must follow all instruction definitions as they use the names
16529 // defined in the instructions definitions.
16530
16531 // Local Variables:
16532 // mode: c++
16533 // End: