1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1713
1714 //=============================================================================
1715
1716 #ifndef PRODUCT
1717 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1718 st->print("nop \t# %d bytes pad for loops and calls", _count);
1719 }
1720 #endif
1721
1722 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1723 for (int i = 0; i < _count; i++) {
1724 __ nop();
1725 }
1726 }
1727
1728 uint MachNopNode::size(PhaseRegAlloc*) const {
1729 return _count * NativeInstruction::instruction_size;
1730 }
1731
1732 //=============================================================================
1733 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1734
1735 int ConstantTable::calculate_table_base_offset() const {
1736 return 0; // absolute addressing, no offset
1737 }
1738
1739 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1740 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1741 ShouldNotReachHere();
1742 }
1743
1744 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1745 // Empty encoding
1746 }
1747
1748 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1749 return 0;
1750 }
1751
1752 #ifndef PRODUCT
1753 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1754 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1755 }
1756 #endif
1757
1758 #ifndef PRODUCT
1759 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1760 Compile* C = ra_->C;
1761
1762 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1763
1764 if (C->output()->need_stack_bang(framesize))
1765 st->print("# stack bang size=%d\n\t", framesize);
1766
1767 if (VM_Version::use_rop_protection()) {
1768 st->print("ldr zr, [lr]\n\t");
1769 st->print("paciaz\n\t");
1770 }
1771 if (framesize < ((1 << 9) + 2 * wordSize)) {
1772 st->print("sub sp, sp, #%d\n\t", framesize);
1773 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1774 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1775 } else {
1776 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1777 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1778 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1779 st->print("sub sp, sp, rscratch1");
1780 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1886 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1887 st->print("add sp, sp, rscratch1\n\t");
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
1952 if (reg < slots_of_int_registers) {
1953 return rc_int;
1954 }
1955
1956 // we have 32 float register * 8 halves
1957 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1958 if (reg < slots_of_int_registers + slots_of_float_registers) {
1959 return rc_float;
1960 }
1961
1962 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1963 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1964 return rc_predicate;
1965 }
1966
1967 // Between predicate regs & stack is the flags.
1968 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1969
1970 return rc_stack;
1971 }
1972
1973 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1974 Compile* C = ra_->C;
1975
1976 // Get registers to move.
1977 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1978 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1979 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1980 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1981
1982 enum RC src_hi_rc = rc_class(src_hi);
1983 enum RC src_lo_rc = rc_class(src_lo);
1984 enum RC dst_hi_rc = rc_class(dst_hi);
1985 enum RC dst_lo_rc = rc_class(dst_lo);
1986
1987 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1988
1989 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1990 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1991 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1992 "expected aligned-adjacent pairs");
1993 }
1994
1995 if (src_lo == dst_lo && src_hi == dst_hi) {
1996 return 0; // Self copy, no move.
1997 }
1998
1999 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2000 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2001 int src_offset = ra_->reg2offset(src_lo);
2002 int dst_offset = ra_->reg2offset(dst_lo);
2003
2004 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2005 uint ireg = ideal_reg();
2006 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2007 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2008 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2009 if (ireg == Op_VecA && masm) {
2010 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2011 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2012 // stack->stack
2013 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2016 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2017 sve_vector_reg_size_in_bytes);
2018 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2019 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2020 sve_vector_reg_size_in_bytes);
2021 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2022 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2023 as_FloatRegister(Matcher::_regEncode[src_lo]),
2024 as_FloatRegister(Matcher::_regEncode[src_lo]));
2025 } else {
2026 ShouldNotReachHere();
2027 }
2028 } else if (masm) {
2029 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2030 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2031 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2032 // stack->stack
2033 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2034 if (ireg == Op_VecD) {
2035 __ unspill(rscratch1, true, src_offset);
2036 __ spill(rscratch1, true, dst_offset);
2037 } else {
2038 __ spill_copy128(src_offset, dst_offset);
2039 }
2040 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2041 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 ireg == Op_VecD ? __ T8B : __ T16B,
2043 as_FloatRegister(Matcher::_regEncode[src_lo]));
2044 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2045 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2046 ireg == Op_VecD ? __ D : __ Q,
2047 ra_->reg2offset(dst_lo));
2048 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2049 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 ireg == Op_VecD ? __ D : __ Q,
2051 ra_->reg2offset(src_lo));
2052 } else {
2053 ShouldNotReachHere();
2054 }
2055 }
2056 } else if (masm) {
2057 switch (src_lo_rc) {
2058 case rc_int:
2059 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2060 if (is64) {
2061 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2062 as_Register(Matcher::_regEncode[src_lo]));
2063 } else {
2064 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2065 as_Register(Matcher::_regEncode[src_lo]));
2066 }
2067 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2068 if (is64) {
2069 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2070 as_Register(Matcher::_regEncode[src_lo]));
2071 } else {
2072 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 as_Register(Matcher::_regEncode[src_lo]));
2074 }
2075 } else { // gpr --> stack spill
2076 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2077 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2078 }
2079 break;
2080 case rc_float:
2081 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2082 if (is64) {
2083 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2084 as_FloatRegister(Matcher::_regEncode[src_lo]));
2085 } else {
2086 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2087 as_FloatRegister(Matcher::_regEncode[src_lo]));
2088 }
2089 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2090 if (is64) {
2091 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2092 as_FloatRegister(Matcher::_regEncode[src_lo]));
2093 } else {
2094 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 }
2097 } else { // fpr --> stack spill
2098 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2099 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2100 is64 ? __ D : __ S, dst_offset);
2101 }
2102 break;
2103 case rc_stack:
2104 if (dst_lo_rc == rc_int) { // stack --> gpr load
2105 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2106 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2107 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2108 is64 ? __ D : __ S, src_offset);
2109 } else if (dst_lo_rc == rc_predicate) {
2110 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2111 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2112 } else { // stack --> stack copy
2113 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2114 if (ideal_reg() == Op_RegVectMask) {
2115 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2116 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2117 } else {
2118 __ unspill(rscratch1, is64, src_offset);
2119 __ spill(rscratch1, is64, dst_offset);
2120 }
2121 }
2122 break;
2123 case rc_predicate:
2124 if (dst_lo_rc == rc_predicate) {
2125 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2126 } else if (dst_lo_rc == rc_stack) {
2127 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2128 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2129 } else {
2130 assert(false, "bad src and dst rc_class combination.");
2131 ShouldNotReachHere();
2132 }
2133 break;
2134 default:
2135 assert(false, "bad rc_class for spill");
2136 ShouldNotReachHere();
2137 }
2138 }
2139
2140 if (st) {
2141 st->print("spill ");
2142 if (src_lo_rc == rc_stack) {
2143 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2144 } else {
2145 st->print("%s -> ", Matcher::regName[src_lo]);
2146 }
2147 if (dst_lo_rc == rc_stack) {
2148 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2149 } else {
2150 st->print("%s", Matcher::regName[dst_lo]);
2151 }
2152 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2153 int vsize = 0;
2154 switch (ideal_reg()) {
2155 case Op_VecD:
2156 vsize = 64;
2157 break;
2158 case Op_VecX:
2159 vsize = 128;
2160 break;
2161 case Op_VecA:
2162 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2163 break;
2164 default:
2165 assert(false, "bad register type for spill");
2166 ShouldNotReachHere();
2167 }
2168 st->print("\t# vector spill size = %d", vsize);
2169 } else if (ideal_reg() == Op_RegVectMask) {
2170 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2171 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2172 st->print("\t# predicate spill size = %d", vsize);
2173 } else {
2174 st->print("\t# spill size = %d", is64 ? 64 : 32);
2175 }
2176 }
2177
2178 return 0;
2179
2180 }
2181
2182 #ifndef PRODUCT
2183 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2184 if (!ra_)
2185 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2186 else
2187 implementation(nullptr, ra_, false, st);
2188 }
2189 #endif
2190
2191 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2192 implementation(masm, ra_, false, nullptr);
2193 }
2194
2195 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2196 return MachNode::size(ra_);
2197 }
2198
2199 //=============================================================================
2200
2201 #ifndef PRODUCT
2202 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2203 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2204 int reg = ra_->get_reg_first(this);
2205 st->print("add %s, rsp, #%d]\t# box lock",
2206 Matcher::regName[reg], offset);
2207 }
2208 #endif
2209
2210 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2211 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2212 int reg = ra_->get_encode(this);
2213
2214 // This add will handle any 24-bit signed offset. 24 bits allows an
2215 // 8 megabyte stack frame.
2216 __ add(as_Register(reg), sp, offset);
2217 }
2218
2219 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2220 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2221 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2222
2223 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2224 return NativeInstruction::instruction_size;
2225 } else {
2226 return 2 * NativeInstruction::instruction_size;
2227 }
2228 }
2229
2230 //=============================================================================
2231
2232 #ifndef PRODUCT
2233 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2234 {
2235 st->print_cr("# MachUEPNode");
2236 if (UseCompressedClassPointers) {
2237 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2238 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2239 st->print_cr("\tcmpw rscratch1, r10");
2240 } else {
2241 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2242 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2243 st->print_cr("\tcmp rscratch1, r10");
2244 }
2245 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2246 }
2247 #endif
2248
2249 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2250 {
2251 __ ic_check(InteriorEntryAlignment);
2252 }
2253
2254 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2255 {
2256 return MachNode::size(ra_);
2257 }
2258
2259 // REQUIRED EMIT CODE
2260
2261 //=============================================================================
2262
2263 // Emit deopt handler code.
2264 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2265 {
2266 // Note that the code buffer's insts_mark is always relative to insts.
2267 // That's why we must use the macroassembler to generate a handler.
2268 address base = __ start_a_stub(size_deopt_handler());
2269 if (base == nullptr) {
2270 ciEnv::current()->record_failure("CodeCache is full");
2271 return 0; // CodeBuffer::expand failed
2272 }
2273
2274 int offset = __ offset();
2275 Label start;
2276 __ bind(start);
2277 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2278
2279 int entry_offset = __ offset();
2280 __ b(start);
2281
2282 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2283 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2284 "out of bounds read in post-call NOP check");
2285 __ end_a_stub();
2286 return entry_offset;
2287 }
2288
2289 // REQUIRED MATCHER CODE
2290
2291 //=============================================================================
2292
2293 bool Matcher::match_rule_supported(int opcode) {
2294 if (!has_match_rule(opcode))
2295 return false;
2296
2297 switch (opcode) {
2298 case Op_OnSpinWait:
2299 return VM_Version::supports_on_spin_wait();
2300 case Op_CacheWB:
2301 case Op_CacheWBPreSync:
2302 case Op_CacheWBPostSync:
2303 if (!VM_Version::supports_data_cache_line_flush()) {
2304 return false;
2305 }
2306 break;
2307 case Op_ExpandBits:
2308 case Op_CompressBits:
2309 if (!VM_Version::supports_svebitperm()) {
2310 return false;
2311 }
2312 break;
2313 case Op_FmaF:
2314 case Op_FmaD:
2315 case Op_FmaVF:
2316 case Op_FmaVD:
2317 if (!UseFMA) {
2318 return false;
2319 }
2320 break;
2321 case Op_FmaHF:
2322 // UseFMA flag also needs to be checked along with FEAT_FP16
2323 if (!UseFMA || !is_feat_fp16_supported()) {
2324 return false;
2325 }
2326 break;
2327 case Op_AddHF:
2328 case Op_SubHF:
2329 case Op_MulHF:
2330 case Op_DivHF:
2331 case Op_MinHF:
2332 case Op_MaxHF:
2333 case Op_SqrtHF:
2334 // Half-precision floating point scalar operations require FEAT_FP16
2335 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2336 // features are supported.
2337 if (!is_feat_fp16_supported()) {
2338 return false;
2339 }
2340 break;
2341 }
2342
2343 return true; // Per default match rules are supported.
2344 }
2345
2346 const RegMask* Matcher::predicate_reg_mask(void) {
2347 return &_PR_REG_mask;
2348 }
2349
2350 bool Matcher::supports_vector_calling_convention(void) {
2351 return EnableVectorSupport;
2352 }
2353
2354 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2355 assert(EnableVectorSupport, "sanity");
2356 int lo = V0_num;
2357 int hi = V0_H_num;
2358 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2359 hi = V0_K_num;
2360 }
2361 return OptoRegPair(hi, lo);
2362 }
2363
2364 // Is this branch offset short enough that a short branch can be used?
2365 //
2366 // NOTE: If the platform does not provide any short branch variants, then
2367 // this method should return false for offset 0.
2368 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2369 // The passed offset is relative to address of the branch.
2370
2371 return (-32768 <= offset && offset < 32768);
2372 }
2373
2374 // Vector width in bytes.
2375 int Matcher::vector_width_in_bytes(BasicType bt) {
2376 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2377 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2378 // Minimum 2 values in vector
2379 if (size < 2*type2aelembytes(bt)) size = 0;
2380 // But never < 4
2381 if (size < 4) size = 0;
2382 return size;
2383 }
2384
2385 // Limits on vector size (number of elements) loaded into vector.
2386 int Matcher::max_vector_size(const BasicType bt) {
2387 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2388 }
2389
2390 int Matcher::min_vector_size(const BasicType bt) {
2391 // Usually, the shortest vector length supported by AArch64 ISA and
2392 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2393 // vectors in a few special cases.
2394 int size;
2395 switch(bt) {
2396 case T_BOOLEAN:
2397 // Load/store a vector mask with only 2 elements for vector types
2398 // such as "2I/2F/2L/2D".
2399 size = 2;
2400 break;
2401 case T_BYTE:
2402 // Generate a "4B" vector, to support vector cast between "8B/16B"
2403 // and "4S/4I/4L/4F/4D".
2404 size = 4;
2405 break;
2406 case T_SHORT:
2407 // Generate a "2S" vector, to support vector cast between "4S/8S"
2408 // and "2I/2L/2F/2D".
2409 size = 2;
2410 break;
2411 default:
2412 // Limit the min vector length to 64-bit.
2413 size = 8 / type2aelembytes(bt);
2414 // The number of elements in a vector should be at least 2.
2415 size = MAX2(size, 2);
2416 }
2417
2418 int max_size = max_vector_size(bt);
2419 return MIN2(size, max_size);
2420 }
2421
2422 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2423 return Matcher::max_vector_size(bt);
2424 }
2425
2426 // Actual max scalable vector register length.
2427 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2428 return Matcher::max_vector_size(bt);
2429 }
2430
2431 // Vector ideal reg.
2432 uint Matcher::vector_ideal_reg(int len) {
2433 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2434 return Op_VecA;
2435 }
2436 switch(len) {
2437 // For 16-bit/32-bit mask vector, reuse VecD.
2438 case 2:
2439 case 4:
2440 case 8: return Op_VecD;
2441 case 16: return Op_VecX;
2442 }
2443 ShouldNotReachHere();
2444 return 0;
2445 }
2446
2447 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2448 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2449 switch (ideal_reg) {
2450 case Op_VecA: return new vecAOper();
2451 case Op_VecD: return new vecDOper();
2452 case Op_VecX: return new vecXOper();
2453 }
2454 ShouldNotReachHere();
2455 return nullptr;
2456 }
2457
2458 bool Matcher::is_reg2reg_move(MachNode* m) {
2459 return false;
2460 }
2461
2462 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2463 return false;
2464 }
2465
2466 bool Matcher::is_generic_vector(MachOper* opnd) {
2467 return opnd->opcode() == VREG;
2468 }
2469
2470 #ifdef ASSERT
2471 // Return whether or not this register is ever used as an argument.
2472 bool Matcher::can_be_java_arg(int reg)
2473 {
2474 return
2475 reg == R0_num || reg == R0_H_num ||
2476 reg == R1_num || reg == R1_H_num ||
2477 reg == R2_num || reg == R2_H_num ||
2478 reg == R3_num || reg == R3_H_num ||
2479 reg == R4_num || reg == R4_H_num ||
2480 reg == R5_num || reg == R5_H_num ||
2481 reg == R6_num || reg == R6_H_num ||
2482 reg == R7_num || reg == R7_H_num ||
2483 reg == V0_num || reg == V0_H_num ||
2484 reg == V1_num || reg == V1_H_num ||
2485 reg == V2_num || reg == V2_H_num ||
2486 reg == V3_num || reg == V3_H_num ||
2487 reg == V4_num || reg == V4_H_num ||
2488 reg == V5_num || reg == V5_H_num ||
2489 reg == V6_num || reg == V6_H_num ||
2490 reg == V7_num || reg == V7_H_num;
2491 }
2492 #endif
2493
2494 uint Matcher::int_pressure_limit()
2495 {
2496 // JDK-8183543: When taking the number of available registers as int
2497 // register pressure threshold, the jtreg test:
2498 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2499 // failed due to C2 compilation failure with
2500 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2501 //
2502 // A derived pointer is live at CallNode and then is flagged by RA
2503 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2504 // derived pointers and lastly fail to spill after reaching maximum
2505 // number of iterations. Lowering the default pressure threshold to
2506 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2507 // a high register pressure area of the code so that split_DEF can
2508 // generate DefinitionSpillCopy for the derived pointer.
2509 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2510 if (!PreserveFramePointer) {
2511 // When PreserveFramePointer is off, frame pointer is allocatable,
2512 // but different from other SOC registers, it is excluded from
2513 // fatproj's mask because its save type is No-Save. Decrease 1 to
2514 // ensure high pressure at fatproj when PreserveFramePointer is off.
2515 // See check_pressure_at_fatproj().
2516 default_int_pressure_threshold--;
2517 }
2518 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2519 }
2520
2521 uint Matcher::float_pressure_limit()
2522 {
2523 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2524 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2525 }
2526
2527 const RegMask& Matcher::divI_proj_mask() {
2528 ShouldNotReachHere();
2529 return RegMask::EMPTY;
2530 }
2531
2532 // Register for MODI projection of divmodI.
2533 const RegMask& Matcher::modI_proj_mask() {
2534 ShouldNotReachHere();
2535 return RegMask::EMPTY;
2536 }
2537
2538 // Register for DIVL projection of divmodL.
2539 const RegMask& Matcher::divL_proj_mask() {
2540 ShouldNotReachHere();
2541 return RegMask::EMPTY;
2542 }
2543
2544 // Register for MODL projection of divmodL.
2545 const RegMask& Matcher::modL_proj_mask() {
2546 ShouldNotReachHere();
2547 return RegMask::EMPTY;
2548 }
2549
2550 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2551 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2552 Node* u = addp->fast_out(i);
2553 if (u->is_LoadStore()) {
2554 // On AArch64, LoadStoreNodes (i.e. compare and swap
2555 // instructions) only take register indirect as an operand, so
2556 // any attempt to use an AddPNode as an input to a LoadStoreNode
2557 // must fail.
2558 return false;
2559 }
2560 if (u->is_Mem()) {
2561 int opsize = u->as_Mem()->memory_size();
2562 assert(opsize > 0, "unexpected memory operand size");
2563 if (u->as_Mem()->memory_size() != (1<<shift)) {
2564 return false;
2565 }
2566 }
2567 }
2568 return true;
2569 }
2570
2571 // Convert BoolTest condition to Assembler condition.
2572 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2573 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2574 Assembler::Condition result;
2575 switch(cond) {
2576 case BoolTest::eq:
2577 result = Assembler::EQ; break;
2578 case BoolTest::ne:
2579 result = Assembler::NE; break;
2580 case BoolTest::le:
2581 result = Assembler::LE; break;
2582 case BoolTest::ge:
2583 result = Assembler::GE; break;
2584 case BoolTest::lt:
2585 result = Assembler::LT; break;
2586 case BoolTest::gt:
2587 result = Assembler::GT; break;
2588 case BoolTest::ule:
2589 result = Assembler::LS; break;
2590 case BoolTest::uge:
2591 result = Assembler::HS; break;
2592 case BoolTest::ult:
2593 result = Assembler::LO; break;
2594 case BoolTest::ugt:
2595 result = Assembler::HI; break;
2596 case BoolTest::overflow:
2597 result = Assembler::VS; break;
2598 case BoolTest::no_overflow:
2599 result = Assembler::VC; break;
2600 default:
2601 ShouldNotReachHere();
2602 return Assembler::Condition(-1);
2603 }
2604
2605 // Check conversion
2606 if (cond & BoolTest::unsigned_compare) {
2607 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2608 } else {
2609 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2610 }
2611
2612 return result;
2613 }
2614
2615 // Binary src (Replicate con)
2616 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2617 if (n == nullptr || m == nullptr) {
2618 return false;
2619 }
2620
2621 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2622 return false;
2623 }
2624
2625 Node* imm_node = m->in(1);
2626 if (!imm_node->is_Con()) {
2627 return false;
2628 }
2629
2630 const Type* t = imm_node->bottom_type();
2631 if (!(t->isa_int() || t->isa_long())) {
2632 return false;
2633 }
2634
2635 switch (n->Opcode()) {
2636 case Op_AndV:
2637 case Op_OrV:
2638 case Op_XorV: {
2639 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2640 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2641 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2642 }
2643 case Op_AddVB:
2644 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2645 case Op_AddVS:
2646 case Op_AddVI:
2647 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2648 case Op_AddVL:
2649 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2650 default:
2651 return false;
2652 }
2653 }
2654
2655 // (XorV src (Replicate m1))
2656 // (XorVMask src (MaskAll m1))
2657 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2658 if (n != nullptr && m != nullptr) {
2659 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2660 VectorNode::is_all_ones_vector(m);
2661 }
2662 return false;
2663 }
2664
2665 // Should the matcher clone input 'm' of node 'n'?
2666 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2667 if (is_vshift_con_pattern(n, m) ||
2668 is_vector_bitwise_not_pattern(n, m) ||
2669 is_valid_sve_arith_imm_pattern(n, m) ||
2670 is_encode_and_store_pattern(n, m)) {
2671 mstack.push(m, Visit);
2672 return true;
2673 }
2674 return false;
2675 }
2676
2677 // Should the Matcher clone shifts on addressing modes, expecting them
2678 // to be subsumed into complex addressing expressions or compute them
2679 // into registers?
2680 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2681
2682 // Loads and stores with indirect memory input (e.g., volatile loads and
2683 // stores) do not subsume the input into complex addressing expressions. If
2684 // the addressing expression is input to at least one such load or store, do
2685 // not clone the addressing expression. Query needs_acquiring_load and
2686 // needs_releasing_store as a proxy for indirect memory input, as it is not
2687 // possible to directly query for indirect memory input at this stage.
2688 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2689 Node* n = m->fast_out(i);
2690 if (n->is_Load() && needs_acquiring_load(n)) {
2691 return false;
2692 }
2693 if (n->is_Store() && needs_releasing_store(n)) {
2694 return false;
2695 }
2696 }
2697
2698 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2699 return true;
2700 }
2701
2702 Node *off = m->in(AddPNode::Offset);
2703 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2704 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2705 // Are there other uses besides address expressions?
2706 !is_visited(off)) {
2707 address_visited.set(off->_idx); // Flag as address_visited
2708 mstack.push(off->in(2), Visit);
2709 Node *conv = off->in(1);
2710 if (conv->Opcode() == Op_ConvI2L &&
2711 // Are there other uses besides address expressions?
2712 !is_visited(conv)) {
2713 address_visited.set(conv->_idx); // Flag as address_visited
2714 mstack.push(conv->in(1), Pre_Visit);
2715 } else {
2716 mstack.push(conv, Pre_Visit);
2717 }
2718 address_visited.test_set(m->_idx); // Flag as address_visited
2719 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2720 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2721 return true;
2722 } else if (off->Opcode() == Op_ConvI2L &&
2723 // Are there other uses besides address expressions?
2724 !is_visited(off)) {
2725 address_visited.test_set(m->_idx); // Flag as address_visited
2726 address_visited.set(off->_idx); // Flag as address_visited
2727 mstack.push(off->in(1), Pre_Visit);
2728 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2729 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2730 return true;
2731 }
2732 return false;
2733 }
2734
2735 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2736 { \
2737 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2738 guarantee(DISP == 0, "mode not permitted for volatile"); \
2739 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2740 __ INSN(REG, as_Register(BASE)); \
2741 }
2742
2743
2744 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2745 {
2746 Address::extend scale;
2747
2748 // Hooboy, this is fugly. We need a way to communicate to the
2749 // encoder that the index needs to be sign extended, so we have to
2750 // enumerate all the cases.
2751 switch (opcode) {
2752 case INDINDEXSCALEDI2L:
2753 case INDINDEXSCALEDI2LN:
2754 case INDINDEXI2L:
2755 case INDINDEXI2LN:
2756 scale = Address::sxtw(size);
2757 break;
2758 default:
2759 scale = Address::lsl(size);
2760 }
2761
2762 if (index == -1) {
2763 return Address(base, disp);
2764 } else {
2765 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2766 return Address(base, as_Register(index), scale);
2767 }
2768 }
2769
2770
2771 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2772 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2773 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2774 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2775 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2776
2777 // Used for all non-volatile memory accesses. The use of
2778 // $mem->opcode() to discover whether this pattern uses sign-extended
2779 // offsets is something of a kludge.
2780 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2781 Register reg, int opcode,
2782 Register base, int index, int scale, int disp,
2783 int size_in_memory)
2784 {
2785 Address addr = mem2address(opcode, base, index, scale, disp);
2786 if (addr.getMode() == Address::base_plus_offset) {
2787 /* Fix up any out-of-range offsets. */
2788 assert_different_registers(rscratch1, base);
2789 assert_different_registers(rscratch1, reg);
2790 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2791 }
2792 (masm->*insn)(reg, addr);
2793 }
2794
2795 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2796 FloatRegister reg, int opcode,
2797 Register base, int index, int size, int disp,
2798 int size_in_memory)
2799 {
2800 Address::extend scale;
2801
2802 switch (opcode) {
2803 case INDINDEXSCALEDI2L:
2804 case INDINDEXSCALEDI2LN:
2805 scale = Address::sxtw(size);
2806 break;
2807 default:
2808 scale = Address::lsl(size);
2809 }
2810
2811 if (index == -1) {
2812 // Fix up any out-of-range offsets.
2813 assert_different_registers(rscratch1, base);
2814 Address addr = Address(base, disp);
2815 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2816 (masm->*insn)(reg, addr);
2817 } else {
2818 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2819 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2820 }
2821 }
2822
2823 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2824 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2825 int opcode, Register base, int index, int size, int disp)
2826 {
2827 if (index == -1) {
2828 (masm->*insn)(reg, T, Address(base, disp));
2829 } else {
2830 assert(disp == 0, "unsupported address mode");
2831 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2832 }
2833 }
2834
2835 %}
2836
2837
2838
2839 //----------ENCODING BLOCK-----------------------------------------------------
2840 // This block specifies the encoding classes used by the compiler to
2841 // output byte streams. Encoding classes are parameterized macros
2842 // used by Machine Instruction Nodes in order to generate the bit
2843 // encoding of the instruction. Operands specify their base encoding
2844 // interface with the interface keyword. There are currently
2845 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2846 // COND_INTER. REG_INTER causes an operand to generate a function
2847 // which returns its register number when queried. CONST_INTER causes
2848 // an operand to generate a function which returns the value of the
2849 // constant when queried. MEMORY_INTER causes an operand to generate
2850 // four functions which return the Base Register, the Index Register,
2851 // the Scale Value, and the Offset Value of the operand when queried.
2852 // COND_INTER causes an operand to generate six functions which return
2853 // the encoding code (ie - encoding bits for the instruction)
2854 // associated with each basic boolean condition for a conditional
2855 // instruction.
2856 //
2857 // Instructions specify two basic values for encoding. Again, a
2858 // function is available to check if the constant displacement is an
2859 // oop. They use the ins_encode keyword to specify their encoding
2860 // classes (which must be a sequence of enc_class names, and their
2861 // parameters, specified in the encoding block), and they use the
2862 // opcode keyword to specify, in order, their primary, secondary, and
2863 // tertiary opcode. Only the opcode sections which a particular
2864 // instruction needs for encoding need to be specified.
2865 encode %{
2866 // Build emit functions for each basic byte or larger field in the
2867 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2868 // from C++ code in the enc_class source block. Emit functions will
2869 // live in the main source block for now. In future, we can
2870 // generalize this by adding a syntax that specifies the sizes of
2871 // fields in an order, so that the adlc can build the emit functions
2872 // automagically
2873
2874 // catch all for unimplemented encodings
2875 enc_class enc_unimplemented %{
2876 __ unimplemented("C2 catch all");
2877 %}
2878
2879 // BEGIN Non-volatile memory access
2880
2881 // This encoding class is generated automatically from ad_encode.m4.
2882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2883 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2884 Register dst_reg = as_Register($dst$$reg);
2885 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2886 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2887 %}
2888
2889 // This encoding class is generated automatically from ad_encode.m4.
2890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2891 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2892 Register dst_reg = as_Register($dst$$reg);
2893 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2894 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2895 %}
2896
2897 // This encoding class is generated automatically from ad_encode.m4.
2898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2899 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2900 Register dst_reg = as_Register($dst$$reg);
2901 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2902 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2903 %}
2904
2905 // This encoding class is generated automatically from ad_encode.m4.
2906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2907 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2908 Register dst_reg = as_Register($dst$$reg);
2909 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2910 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2911 %}
2912
2913 // This encoding class is generated automatically from ad_encode.m4.
2914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2915 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2916 Register dst_reg = as_Register($dst$$reg);
2917 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2918 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2919 %}
2920
2921 // This encoding class is generated automatically from ad_encode.m4.
2922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2923 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2924 Register dst_reg = as_Register($dst$$reg);
2925 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2926 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2927 %}
2928
2929 // This encoding class is generated automatically from ad_encode.m4.
2930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2931 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2932 Register dst_reg = as_Register($dst$$reg);
2933 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2934 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2935 %}
2936
2937 // This encoding class is generated automatically from ad_encode.m4.
2938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2939 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2940 Register dst_reg = as_Register($dst$$reg);
2941 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2942 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2943 %}
2944
2945 // This encoding class is generated automatically from ad_encode.m4.
2946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2947 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2948 Register dst_reg = as_Register($dst$$reg);
2949 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2950 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2951 %}
2952
2953 // This encoding class is generated automatically from ad_encode.m4.
2954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2955 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2956 Register dst_reg = as_Register($dst$$reg);
2957 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2958 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2959 %}
2960
2961 // This encoding class is generated automatically from ad_encode.m4.
2962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2963 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2964 Register dst_reg = as_Register($dst$$reg);
2965 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2966 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2967 %}
2968
2969 // This encoding class is generated automatically from ad_encode.m4.
2970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2971 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2972 Register dst_reg = as_Register($dst$$reg);
2973 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2974 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2975 %}
2976
2977 // This encoding class is generated automatically from ad_encode.m4.
2978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2979 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2980 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2981 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2982 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2983 %}
2984
2985 // This encoding class is generated automatically from ad_encode.m4.
2986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2987 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2988 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2989 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2990 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2991 %}
2992
2993 // This encoding class is generated automatically from ad_encode.m4.
2994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2995 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2996 Register src_reg = as_Register($src$$reg);
2997 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2998 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2999 %}
3000
3001 // This encoding class is generated automatically from ad_encode.m4.
3002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3003 enc_class aarch64_enc_strb0(memory1 mem) %{
3004 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3005 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3006 %}
3007
3008 // This encoding class is generated automatically from ad_encode.m4.
3009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3010 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3011 Register src_reg = as_Register($src$$reg);
3012 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3013 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3014 %}
3015
3016 // This encoding class is generated automatically from ad_encode.m4.
3017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3018 enc_class aarch64_enc_strh0(memory2 mem) %{
3019 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3020 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3021 %}
3022
3023 // This encoding class is generated automatically from ad_encode.m4.
3024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3025 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3026 Register src_reg = as_Register($src$$reg);
3027 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3028 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3029 %}
3030
3031 // This encoding class is generated automatically from ad_encode.m4.
3032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3033 enc_class aarch64_enc_strw0(memory4 mem) %{
3034 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3035 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3036 %}
3037
3038 // This encoding class is generated automatically from ad_encode.m4.
3039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3040 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3041 Register src_reg = as_Register($src$$reg);
3042 // we sometimes get asked to store the stack pointer into the
3043 // current thread -- we cannot do that directly on AArch64
3044 if (src_reg == r31_sp) {
3045 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3046 __ mov(rscratch2, sp);
3047 src_reg = rscratch2;
3048 }
3049 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3050 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3051 %}
3052
3053 // This encoding class is generated automatically from ad_encode.m4.
3054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3055 enc_class aarch64_enc_str0(memory8 mem) %{
3056 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3057 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3058 %}
3059
3060 // This encoding class is generated automatically from ad_encode.m4.
3061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3062 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3063 FloatRegister src_reg = as_FloatRegister($src$$reg);
3064 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3065 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3066 %}
3067
3068 // This encoding class is generated automatically from ad_encode.m4.
3069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3070 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3071 FloatRegister src_reg = as_FloatRegister($src$$reg);
3072 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3073 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3074 %}
3075
3076 // This encoding class is generated automatically from ad_encode.m4.
3077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3078 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3079 __ membar(Assembler::StoreStore);
3080 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3081 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3082 %}
3083
3084 // END Non-volatile memory access
3085
3086 // Vector loads and stores
3087 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3088 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3089 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3090 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3091 %}
3092
3093 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3094 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3095 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3096 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3097 %}
3098
3099 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3100 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3101 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3102 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3103 %}
3104
3105 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3106 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3107 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3108 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3109 %}
3110
3111 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3112 FloatRegister src_reg = as_FloatRegister($src$$reg);
3113 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3114 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3115 %}
3116
3117 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3118 FloatRegister src_reg = as_FloatRegister($src$$reg);
3119 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3120 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3121 %}
3122
3123 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3124 FloatRegister src_reg = as_FloatRegister($src$$reg);
3125 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3126 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3127 %}
3128
3129 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3130 FloatRegister src_reg = as_FloatRegister($src$$reg);
3131 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3132 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3133 %}
3134
3135 // volatile loads and stores
3136
3137 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3138 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3139 rscratch1, stlrb);
3140 %}
3141
3142 enc_class aarch64_enc_stlrb0(memory mem) %{
3143 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3144 rscratch1, stlrb);
3145 %}
3146
3147 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3148 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, stlrh);
3150 %}
3151
3152 enc_class aarch64_enc_stlrh0(memory mem) %{
3153 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, stlrh);
3155 %}
3156
3157 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3158 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3159 rscratch1, stlrw);
3160 %}
3161
3162 enc_class aarch64_enc_stlrw0(memory mem) %{
3163 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, stlrw);
3165 %}
3166
3167 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3168 Register dst_reg = as_Register($dst$$reg);
3169 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3170 rscratch1, ldarb);
3171 __ sxtbw(dst_reg, dst_reg);
3172 %}
3173
3174 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3175 Register dst_reg = as_Register($dst$$reg);
3176 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3177 rscratch1, ldarb);
3178 __ sxtb(dst_reg, dst_reg);
3179 %}
3180
3181 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3182 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3183 rscratch1, ldarb);
3184 %}
3185
3186 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3187 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3188 rscratch1, ldarb);
3189 %}
3190
3191 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3192 Register dst_reg = as_Register($dst$$reg);
3193 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, ldarh);
3195 __ sxthw(dst_reg, dst_reg);
3196 %}
3197
3198 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3199 Register dst_reg = as_Register($dst$$reg);
3200 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3201 rscratch1, ldarh);
3202 __ sxth(dst_reg, dst_reg);
3203 %}
3204
3205 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3206 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3207 rscratch1, ldarh);
3208 %}
3209
3210 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3211 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, ldarh);
3213 %}
3214
3215 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3216 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3217 rscratch1, ldarw);
3218 %}
3219
3220 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3221 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3222 rscratch1, ldarw);
3223 %}
3224
3225 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3226 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3227 rscratch1, ldar);
3228 %}
3229
3230 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3231 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, ldarw);
3233 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3234 %}
3235
3236 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3237 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3238 rscratch1, ldar);
3239 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3240 %}
3241
3242 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3243 Register src_reg = as_Register($src$$reg);
3244 // we sometimes get asked to store the stack pointer into the
3245 // current thread -- we cannot do that directly on AArch64
3246 if (src_reg == r31_sp) {
3247 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3248 __ mov(rscratch2, sp);
3249 src_reg = rscratch2;
3250 }
3251 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3252 rscratch1, stlr);
3253 %}
3254
3255 enc_class aarch64_enc_stlr0(memory mem) %{
3256 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3257 rscratch1, stlr);
3258 %}
3259
3260 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3261 {
3262 FloatRegister src_reg = as_FloatRegister($src$$reg);
3263 __ fmovs(rscratch2, src_reg);
3264 }
3265 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3266 rscratch1, stlrw);
3267 %}
3268
3269 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3270 {
3271 FloatRegister src_reg = as_FloatRegister($src$$reg);
3272 __ fmovd(rscratch2, src_reg);
3273 }
3274 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3275 rscratch1, stlr);
3276 %}
3277
3278 // synchronized read/update encodings
3279
3280 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3281 Register dst_reg = as_Register($dst$$reg);
3282 Register base = as_Register($mem$$base);
3283 int index = $mem$$index;
3284 int scale = $mem$$scale;
3285 int disp = $mem$$disp;
3286 if (index == -1) {
3287 if (disp != 0) {
3288 __ lea(rscratch1, Address(base, disp));
3289 __ ldaxr(dst_reg, rscratch1);
3290 } else {
3291 // TODO
3292 // should we ever get anything other than this case?
3293 __ ldaxr(dst_reg, base);
3294 }
3295 } else {
3296 Register index_reg = as_Register(index);
3297 if (disp == 0) {
3298 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3299 __ ldaxr(dst_reg, rscratch1);
3300 } else {
3301 __ lea(rscratch1, Address(base, disp));
3302 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3303 __ ldaxr(dst_reg, rscratch1);
3304 }
3305 }
3306 %}
3307
3308 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3309 Register src_reg = as_Register($src$$reg);
3310 Register base = as_Register($mem$$base);
3311 int index = $mem$$index;
3312 int scale = $mem$$scale;
3313 int disp = $mem$$disp;
3314 if (index == -1) {
3315 if (disp != 0) {
3316 __ lea(rscratch2, Address(base, disp));
3317 __ stlxr(rscratch1, src_reg, rscratch2);
3318 } else {
3319 // TODO
3320 // should we ever get anything other than this case?
3321 __ stlxr(rscratch1, src_reg, base);
3322 }
3323 } else {
3324 Register index_reg = as_Register(index);
3325 if (disp == 0) {
3326 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3327 __ stlxr(rscratch1, src_reg, rscratch2);
3328 } else {
3329 __ lea(rscratch2, Address(base, disp));
3330 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3331 __ stlxr(rscratch1, src_reg, rscratch2);
3332 }
3333 }
3334 __ cmpw(rscratch1, zr);
3335 %}
3336
3337 // prefetch encodings
3338
3339 enc_class aarch64_enc_prefetchw(memory mem) %{
3340 Register base = as_Register($mem$$base);
3341 int index = $mem$$index;
3342 int scale = $mem$$scale;
3343 int disp = $mem$$disp;
3344 if (index == -1) {
3345 // Fix up any out-of-range offsets.
3346 assert_different_registers(rscratch1, base);
3347 Address addr = Address(base, disp);
3348 addr = __ legitimize_address(addr, 8, rscratch1);
3349 __ prfm(addr, PSTL1KEEP);
3350 } else {
3351 Register index_reg = as_Register(index);
3352 if (disp == 0) {
3353 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3354 } else {
3355 __ lea(rscratch1, Address(base, disp));
3356 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3357 }
3358 }
3359 %}
3360
3361 // mov encodings
3362
3363 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3364 uint32_t con = (uint32_t)$src$$constant;
3365 Register dst_reg = as_Register($dst$$reg);
3366 if (con == 0) {
3367 __ movw(dst_reg, zr);
3368 } else {
3369 __ movw(dst_reg, con);
3370 }
3371 %}
3372
3373 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3374 Register dst_reg = as_Register($dst$$reg);
3375 uint64_t con = (uint64_t)$src$$constant;
3376 if (con == 0) {
3377 __ mov(dst_reg, zr);
3378 } else {
3379 __ mov(dst_reg, con);
3380 }
3381 %}
3382
3383 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3384 Register dst_reg = as_Register($dst$$reg);
3385 address con = (address)$src$$constant;
3386 if (con == nullptr || con == (address)1) {
3387 ShouldNotReachHere();
3388 } else {
3389 relocInfo::relocType rtype = $src->constant_reloc();
3390 if (rtype == relocInfo::oop_type) {
3391 __ movoop(dst_reg, (jobject)con);
3392 } else if (rtype == relocInfo::metadata_type) {
3393 __ mov_metadata(dst_reg, (Metadata*)con);
3394 } else {
3395 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3396 // load fake address constants using a normal move
3397 if (! __ is_valid_AArch64_address(con) ||
3398 con < (address)(uintptr_t)os::vm_page_size()) {
3399 __ mov(dst_reg, con);
3400 } else {
3401 // no reloc so just use adrp and add
3402 uint64_t offset;
3403 __ adrp(dst_reg, con, offset);
3404 __ add(dst_reg, dst_reg, offset);
3405 }
3406 }
3407 }
3408 %}
3409
3410 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3411 Register dst_reg = as_Register($dst$$reg);
3412 __ mov(dst_reg, zr);
3413 %}
3414
3415 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3416 Register dst_reg = as_Register($dst$$reg);
3417 __ mov(dst_reg, (uint64_t)1);
3418 %}
3419
3420 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3421 Register dst_reg = as_Register($dst$$reg);
3422 address con = (address)$src$$constant;
3423 if (con == nullptr) {
3424 ShouldNotReachHere();
3425 } else {
3426 relocInfo::relocType rtype = $src->constant_reloc();
3427 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3428 __ set_narrow_oop(dst_reg, (jobject)con);
3429 }
3430 %}
3431
3432 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3433 Register dst_reg = as_Register($dst$$reg);
3434 __ mov(dst_reg, zr);
3435 %}
3436
3437 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3438 Register dst_reg = as_Register($dst$$reg);
3439 address con = (address)$src$$constant;
3440 if (con == nullptr) {
3441 ShouldNotReachHere();
3442 } else {
3443 relocInfo::relocType rtype = $src->constant_reloc();
3444 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3445 __ set_narrow_klass(dst_reg, (Klass *)con);
3446 }
3447 %}
3448
3449 // arithmetic encodings
3450
3451 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3452 Register dst_reg = as_Register($dst$$reg);
3453 Register src_reg = as_Register($src1$$reg);
3454 int32_t con = (int32_t)$src2$$constant;
3455 // add has primary == 0, subtract has primary == 1
3456 if ($primary) { con = -con; }
3457 if (con < 0) {
3458 __ subw(dst_reg, src_reg, -con);
3459 } else {
3460 __ addw(dst_reg, src_reg, con);
3461 }
3462 %}
3463
3464 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3465 Register dst_reg = as_Register($dst$$reg);
3466 Register src_reg = as_Register($src1$$reg);
3467 int32_t con = (int32_t)$src2$$constant;
3468 // add has primary == 0, subtract has primary == 1
3469 if ($primary) { con = -con; }
3470 if (con < 0) {
3471 __ sub(dst_reg, src_reg, -con);
3472 } else {
3473 __ add(dst_reg, src_reg, con);
3474 }
3475 %}
3476
3477 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3478 Register dst_reg = as_Register($dst$$reg);
3479 Register src1_reg = as_Register($src1$$reg);
3480 Register src2_reg = as_Register($src2$$reg);
3481 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3482 %}
3483
3484 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3485 Register dst_reg = as_Register($dst$$reg);
3486 Register src1_reg = as_Register($src1$$reg);
3487 Register src2_reg = as_Register($src2$$reg);
3488 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3489 %}
3490
3491 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3492 Register dst_reg = as_Register($dst$$reg);
3493 Register src1_reg = as_Register($src1$$reg);
3494 Register src2_reg = as_Register($src2$$reg);
3495 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3496 %}
3497
3498 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3499 Register dst_reg = as_Register($dst$$reg);
3500 Register src1_reg = as_Register($src1$$reg);
3501 Register src2_reg = as_Register($src2$$reg);
3502 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3503 %}
3504
3505 // compare instruction encodings
3506
3507 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3508 Register reg1 = as_Register($src1$$reg);
3509 Register reg2 = as_Register($src2$$reg);
3510 __ cmpw(reg1, reg2);
3511 %}
3512
3513 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3514 Register reg = as_Register($src1$$reg);
3515 int32_t val = $src2$$constant;
3516 if (val >= 0) {
3517 __ subsw(zr, reg, val);
3518 } else {
3519 __ addsw(zr, reg, -val);
3520 }
3521 %}
3522
3523 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3524 Register reg1 = as_Register($src1$$reg);
3525 uint32_t val = (uint32_t)$src2$$constant;
3526 __ movw(rscratch1, val);
3527 __ cmpw(reg1, rscratch1);
3528 %}
3529
3530 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3531 Register reg1 = as_Register($src1$$reg);
3532 Register reg2 = as_Register($src2$$reg);
3533 __ cmp(reg1, reg2);
3534 %}
3535
3536 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3537 Register reg = as_Register($src1$$reg);
3538 int64_t val = $src2$$constant;
3539 if (val >= 0) {
3540 __ subs(zr, reg, val);
3541 } else if (val != -val) {
3542 __ adds(zr, reg, -val);
3543 } else {
3544 // aargh, Long.MIN_VALUE is a special case
3545 __ orr(rscratch1, zr, (uint64_t)val);
3546 __ subs(zr, reg, rscratch1);
3547 }
3548 %}
3549
3550 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3551 Register reg1 = as_Register($src1$$reg);
3552 uint64_t val = (uint64_t)$src2$$constant;
3553 __ mov(rscratch1, val);
3554 __ cmp(reg1, rscratch1);
3555 %}
3556
3557 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3558 Register reg1 = as_Register($src1$$reg);
3559 Register reg2 = as_Register($src2$$reg);
3560 __ cmp(reg1, reg2);
3561 %}
3562
3563 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3564 Register reg1 = as_Register($src1$$reg);
3565 Register reg2 = as_Register($src2$$reg);
3566 __ cmpw(reg1, reg2);
3567 %}
3568
3569 enc_class aarch64_enc_testp(iRegP src) %{
3570 Register reg = as_Register($src$$reg);
3571 __ cmp(reg, zr);
3572 %}
3573
3574 enc_class aarch64_enc_testn(iRegN src) %{
3575 Register reg = as_Register($src$$reg);
3576 __ cmpw(reg, zr);
3577 %}
3578
3579 enc_class aarch64_enc_b(label lbl) %{
3580 Label *L = $lbl$$label;
3581 __ b(*L);
3582 %}
3583
3584 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3585 Label *L = $lbl$$label;
3586 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3587 %}
3588
3589 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3590 Label *L = $lbl$$label;
3591 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3592 %}
3593
3594 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3595 %{
3596 Register sub_reg = as_Register($sub$$reg);
3597 Register super_reg = as_Register($super$$reg);
3598 Register temp_reg = as_Register($temp$$reg);
3599 Register result_reg = as_Register($result$$reg);
3600
3601 Label miss;
3602 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3603 nullptr, &miss,
3604 /*set_cond_codes:*/ true);
3605 if ($primary) {
3606 __ mov(result_reg, zr);
3607 }
3608 __ bind(miss);
3609 %}
3610
3611 enc_class aarch64_enc_java_static_call(method meth) %{
3612 address addr = (address)$meth$$method;
3613 address call;
3614 if (!_method) {
3615 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3616 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3617 if (call == nullptr) {
3618 ciEnv::current()->record_failure("CodeCache is full");
3619 return;
3620 }
3621 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3622 // The NOP here is purely to ensure that eliding a call to
3623 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3624 __ nop();
3625 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3626 } else {
3627 int method_index = resolved_method_index(masm);
3628 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3629 : static_call_Relocation::spec(method_index);
3630 call = __ trampoline_call(Address(addr, rspec));
3631 if (call == nullptr) {
3632 ciEnv::current()->record_failure("CodeCache is full");
3633 return;
3634 }
3635 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3636 // Calls of the same statically bound method can share
3637 // a stub to the interpreter.
3638 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3639 } else {
3640 // Emit stub for static call
3641 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3642 if (stub == nullptr) {
3643 ciEnv::current()->record_failure("CodeCache is full");
3644 return;
3645 }
3646 }
3647 }
3648
3649 __ post_call_nop();
3650
3651 // Only non uncommon_trap calls need to reinitialize ptrue.
3652 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3653 __ reinitialize_ptrue();
3654 }
3655 %}
3656
3657 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3658 int method_index = resolved_method_index(masm);
3659 address call = __ ic_call((address)$meth$$method, method_index);
3660 if (call == nullptr) {
3661 ciEnv::current()->record_failure("CodeCache is full");
3662 return;
3663 }
3664 __ post_call_nop();
3665 if (Compile::current()->max_vector_size() > 0) {
3666 __ reinitialize_ptrue();
3667 }
3668 %}
3669
3670 enc_class aarch64_enc_call_epilog() %{
3671 if (VerifyStackAtCalls) {
3672 // Check that stack depth is unchanged: find majik cookie on stack
3673 __ call_Unimplemented();
3674 }
3675 %}
3676
3677 enc_class aarch64_enc_java_to_runtime(method meth) %{
3678 // some calls to generated routines (arraycopy code) are scheduled
3679 // by C2 as runtime calls. if so we can call them using a br (they
3680 // will be in a reachable segment) otherwise we have to use a blr
3681 // which loads the absolute address into a register.
3682 address entry = (address)$meth$$method;
3683 CodeBlob *cb = CodeCache::find_blob(entry);
3684 if (cb) {
3685 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3686 if (call == nullptr) {
3687 ciEnv::current()->record_failure("CodeCache is full");
3688 return;
3689 }
3690 __ post_call_nop();
3691 } else {
3692 Label retaddr;
3693 // Make the anchor frame walkable
3694 __ adr(rscratch2, retaddr);
3695 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3696 __ lea(rscratch1, RuntimeAddress(entry));
3697 __ blr(rscratch1);
3698 __ bind(retaddr);
3699 __ post_call_nop();
3700 }
3701 if (Compile::current()->max_vector_size() > 0) {
3702 __ reinitialize_ptrue();
3703 }
3704 %}
3705
3706 enc_class aarch64_enc_rethrow() %{
3707 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3708 %}
3709
3710 enc_class aarch64_enc_ret() %{
3711 #ifdef ASSERT
3712 if (Compile::current()->max_vector_size() > 0) {
3713 __ verify_ptrue();
3714 }
3715 #endif
3716 __ ret(lr);
3717 %}
3718
3719 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3720 Register target_reg = as_Register($jump_target$$reg);
3721 __ br(target_reg);
3722 %}
3723
3724 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3725 Register target_reg = as_Register($jump_target$$reg);
3726 // exception oop should be in r0
3727 // ret addr has been popped into lr
3728 // callee expects it in r3
3729 __ mov(r3, lr);
3730 __ br(target_reg);
3731 %}
3732
3733 %}
3734
3735 //----------FRAME--------------------------------------------------------------
3736 // Definition of frame structure and management information.
3737 //
3738 // S T A C K L A Y O U T Allocators stack-slot number
3739 // | (to get allocators register number
3740 // G Owned by | | v add OptoReg::stack0())
3741 // r CALLER | |
3742 // o | +--------+ pad to even-align allocators stack-slot
3743 // w V | pad0 | numbers; owned by CALLER
3744 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3745 // h ^ | in | 5
3746 // | | args | 4 Holes in incoming args owned by SELF
3747 // | | | | 3
3748 // | | +--------+
3749 // V | | old out| Empty on Intel, window on Sparc
3750 // | old |preserve| Must be even aligned.
3751 // | SP-+--------+----> Matcher::_old_SP, even aligned
3752 // | | in | 3 area for Intel ret address
3753 // Owned by |preserve| Empty on Sparc.
3754 // SELF +--------+
3755 // | | pad2 | 2 pad to align old SP
3756 // | +--------+ 1
3757 // | | locks | 0
3758 // | +--------+----> OptoReg::stack0(), even aligned
3759 // | | pad1 | 11 pad to align new SP
3760 // | +--------+
3761 // | | | 10
3762 // | | spills | 9 spills
3763 // V | | 8 (pad0 slot for callee)
3764 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3765 // ^ | out | 7
3766 // | | args | 6 Holes in outgoing args owned by CALLEE
3767 // Owned by +--------+
3768 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3769 // | new |preserve| Must be even-aligned.
3770 // | SP-+--------+----> Matcher::_new_SP, even aligned
3771 // | | |
3772 //
3773 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3774 // known from SELF's arguments and the Java calling convention.
3775 // Region 6-7 is determined per call site.
3776 // Note 2: If the calling convention leaves holes in the incoming argument
3777 // area, those holes are owned by SELF. Holes in the outgoing area
3778 // are owned by the CALLEE. Holes should not be necessary in the
3779 // incoming area, as the Java calling convention is completely under
3780 // the control of the AD file. Doubles can be sorted and packed to
3781 // avoid holes. Holes in the outgoing arguments may be necessary for
3782 // varargs C calling conventions.
3783 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3784 // even aligned with pad0 as needed.
3785 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3786 // (the latter is true on Intel but is it false on AArch64?)
3787 // region 6-11 is even aligned; it may be padded out more so that
3788 // the region from SP to FP meets the minimum stack alignment.
3789 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3790 // alignment. Region 11, pad1, may be dynamically extended so that
3791 // SP meets the minimum alignment.
3792
3793 frame %{
3794 // These three registers define part of the calling convention
3795 // between compiled code and the interpreter.
3796
3797 // Inline Cache Register or Method for I2C.
3798 inline_cache_reg(R12);
3799
3800 // Number of stack slots consumed by locking an object
3801 sync_stack_slots(2);
3802
3803 // Compiled code's Frame Pointer
3804 frame_pointer(R31);
3805
3806 // Stack alignment requirement
3807 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3808
3809 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3810 // for calls to C. Supports the var-args backing area for register parms.
3811 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3812
3813 // The after-PROLOG location of the return address. Location of
3814 // return address specifies a type (REG or STACK) and a number
3815 // representing the register number (i.e. - use a register name) or
3816 // stack slot.
3817 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3818 // Otherwise, it is above the locks and verification slot and alignment word
3819 // TODO this may well be correct but need to check why that - 2 is there
3820 // ppc port uses 0 but we definitely need to allow for fixed_slots
3821 // which folds in the space used for monitors
3822 return_addr(STACK - 2 +
3823 align_up((Compile::current()->in_preserve_stack_slots() +
3824 Compile::current()->fixed_slots()),
3825 stack_alignment_in_slots()));
3826
3827 // Location of compiled Java return values. Same as C for now.
3828 return_value
3829 %{
3830 // TODO do we allow ideal_reg == Op_RegN???
3831 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3832 "only return normal values");
3833
3834 static const int lo[Op_RegL + 1] = { // enum name
3835 0, // Op_Node
3836 0, // Op_Set
3837 R0_num, // Op_RegN
3838 R0_num, // Op_RegI
3839 R0_num, // Op_RegP
3840 V0_num, // Op_RegF
3841 V0_num, // Op_RegD
3842 R0_num // Op_RegL
3843 };
3844
3845 static const int hi[Op_RegL + 1] = { // enum name
3846 0, // Op_Node
3847 0, // Op_Set
3848 OptoReg::Bad, // Op_RegN
3849 OptoReg::Bad, // Op_RegI
3850 R0_H_num, // Op_RegP
3851 OptoReg::Bad, // Op_RegF
3852 V0_H_num, // Op_RegD
3853 R0_H_num // Op_RegL
3854 };
3855
3856 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3857 %}
3858 %}
3859
3860 //----------ATTRIBUTES---------------------------------------------------------
3861 //----------Operand Attributes-------------------------------------------------
3862 op_attrib op_cost(1); // Required cost attribute
3863
3864 //----------Instruction Attributes---------------------------------------------
3865 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3866 ins_attrib ins_size(32); // Required size attribute (in bits)
3867 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3868 // a non-matching short branch variant
3869 // of some long branch?
3870 ins_attrib ins_alignment(4); // Required alignment attribute (must
3871 // be a power of 2) specifies the
3872 // alignment that some part of the
3873 // instruction (not necessarily the
3874 // start) requires. If > 1, a
3875 // compute_padding() function must be
3876 // provided for the instruction
3877
3878 // Whether this node is expanded during code emission into a sequence of
3879 // instructions and the first instruction can perform an implicit null check.
3880 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3881
3882 //----------OPERANDS-----------------------------------------------------------
3883 // Operand definitions must precede instruction definitions for correct parsing
3884 // in the ADLC because operands constitute user defined types which are used in
3885 // instruction definitions.
3886
3887 //----------Simple Operands----------------------------------------------------
3888
3889 // Integer operands 32 bit
3890 // 32 bit immediate
3891 operand immI()
3892 %{
3893 match(ConI);
3894
3895 op_cost(0);
3896 format %{ %}
3897 interface(CONST_INTER);
3898 %}
3899
3900 // 32 bit zero
3901 operand immI0()
3902 %{
3903 predicate(n->get_int() == 0);
3904 match(ConI);
3905
3906 op_cost(0);
3907 format %{ %}
3908 interface(CONST_INTER);
3909 %}
3910
3911 // 32 bit unit increment
3912 operand immI_1()
3913 %{
3914 predicate(n->get_int() == 1);
3915 match(ConI);
3916
3917 op_cost(0);
3918 format %{ %}
3919 interface(CONST_INTER);
3920 %}
3921
3922 // 32 bit unit decrement
3923 operand immI_M1()
3924 %{
3925 predicate(n->get_int() == -1);
3926 match(ConI);
3927
3928 op_cost(0);
3929 format %{ %}
3930 interface(CONST_INTER);
3931 %}
3932
3933 // Shift values for add/sub extension shift
3934 operand immIExt()
3935 %{
3936 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3937 match(ConI);
3938
3939 op_cost(0);
3940 format %{ %}
3941 interface(CONST_INTER);
3942 %}
3943
3944 operand immI_gt_1()
3945 %{
3946 predicate(n->get_int() > 1);
3947 match(ConI);
3948
3949 op_cost(0);
3950 format %{ %}
3951 interface(CONST_INTER);
3952 %}
3953
3954 operand immI_le_4()
3955 %{
3956 predicate(n->get_int() <= 4);
3957 match(ConI);
3958
3959 op_cost(0);
3960 format %{ %}
3961 interface(CONST_INTER);
3962 %}
3963
3964 operand immI_16()
3965 %{
3966 predicate(n->get_int() == 16);
3967 match(ConI);
3968
3969 op_cost(0);
3970 format %{ %}
3971 interface(CONST_INTER);
3972 %}
3973
3974 operand immI_24()
3975 %{
3976 predicate(n->get_int() == 24);
3977 match(ConI);
3978
3979 op_cost(0);
3980 format %{ %}
3981 interface(CONST_INTER);
3982 %}
3983
3984 operand immI_32()
3985 %{
3986 predicate(n->get_int() == 32);
3987 match(ConI);
3988
3989 op_cost(0);
3990 format %{ %}
3991 interface(CONST_INTER);
3992 %}
3993
3994 operand immI_48()
3995 %{
3996 predicate(n->get_int() == 48);
3997 match(ConI);
3998
3999 op_cost(0);
4000 format %{ %}
4001 interface(CONST_INTER);
4002 %}
4003
4004 operand immI_56()
4005 %{
4006 predicate(n->get_int() == 56);
4007 match(ConI);
4008
4009 op_cost(0);
4010 format %{ %}
4011 interface(CONST_INTER);
4012 %}
4013
4014 operand immI_255()
4015 %{
4016 predicate(n->get_int() == 255);
4017 match(ConI);
4018
4019 op_cost(0);
4020 format %{ %}
4021 interface(CONST_INTER);
4022 %}
4023
4024 operand immI_65535()
4025 %{
4026 predicate(n->get_int() == 65535);
4027 match(ConI);
4028
4029 op_cost(0);
4030 format %{ %}
4031 interface(CONST_INTER);
4032 %}
4033
4034 operand immI_positive()
4035 %{
4036 predicate(n->get_int() > 0);
4037 match(ConI);
4038
4039 op_cost(0);
4040 format %{ %}
4041 interface(CONST_INTER);
4042 %}
4043
4044 // BoolTest condition for signed compare
4045 operand immI_cmp_cond()
4046 %{
4047 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4048 match(ConI);
4049
4050 op_cost(0);
4051 format %{ %}
4052 interface(CONST_INTER);
4053 %}
4054
4055 // BoolTest condition for unsigned compare
4056 operand immI_cmpU_cond()
4057 %{
4058 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4059 match(ConI);
4060
4061 op_cost(0);
4062 format %{ %}
4063 interface(CONST_INTER);
4064 %}
4065
4066 operand immL_255()
4067 %{
4068 predicate(n->get_long() == 255L);
4069 match(ConL);
4070
4071 op_cost(0);
4072 format %{ %}
4073 interface(CONST_INTER);
4074 %}
4075
4076 operand immL_65535()
4077 %{
4078 predicate(n->get_long() == 65535L);
4079 match(ConL);
4080
4081 op_cost(0);
4082 format %{ %}
4083 interface(CONST_INTER);
4084 %}
4085
4086 operand immL_4294967295()
4087 %{
4088 predicate(n->get_long() == 4294967295L);
4089 match(ConL);
4090
4091 op_cost(0);
4092 format %{ %}
4093 interface(CONST_INTER);
4094 %}
4095
4096 operand immL_bitmask()
4097 %{
4098 predicate((n->get_long() != 0)
4099 && ((n->get_long() & 0xc000000000000000l) == 0)
4100 && is_power_of_2(n->get_long() + 1));
4101 match(ConL);
4102
4103 op_cost(0);
4104 format %{ %}
4105 interface(CONST_INTER);
4106 %}
4107
4108 operand immI_bitmask()
4109 %{
4110 predicate((n->get_int() != 0)
4111 && ((n->get_int() & 0xc0000000) == 0)
4112 && is_power_of_2(n->get_int() + 1));
4113 match(ConI);
4114
4115 op_cost(0);
4116 format %{ %}
4117 interface(CONST_INTER);
4118 %}
4119
4120 operand immL_positive_bitmaskI()
4121 %{
4122 predicate((n->get_long() != 0)
4123 && ((julong)n->get_long() < 0x80000000ULL)
4124 && is_power_of_2(n->get_long() + 1));
4125 match(ConL);
4126
4127 op_cost(0);
4128 format %{ %}
4129 interface(CONST_INTER);
4130 %}
4131
4132 // Scale values for scaled offset addressing modes (up to long but not quad)
4133 operand immIScale()
4134 %{
4135 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4136 match(ConI);
4137
4138 op_cost(0);
4139 format %{ %}
4140 interface(CONST_INTER);
4141 %}
4142
4143 // 5 bit signed integer
4144 operand immI5()
4145 %{
4146 predicate(Assembler::is_simm(n->get_int(), 5));
4147 match(ConI);
4148
4149 op_cost(0);
4150 format %{ %}
4151 interface(CONST_INTER);
4152 %}
4153
4154 // 7 bit unsigned integer
4155 operand immIU7()
4156 %{
4157 predicate(Assembler::is_uimm(n->get_int(), 7));
4158 match(ConI);
4159
4160 op_cost(0);
4161 format %{ %}
4162 interface(CONST_INTER);
4163 %}
4164
4165 // Offset for scaled or unscaled immediate loads and stores
4166 operand immIOffset()
4167 %{
4168 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4169 match(ConI);
4170
4171 op_cost(0);
4172 format %{ %}
4173 interface(CONST_INTER);
4174 %}
4175
4176 operand immIOffset1()
4177 %{
4178 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4179 match(ConI);
4180
4181 op_cost(0);
4182 format %{ %}
4183 interface(CONST_INTER);
4184 %}
4185
4186 operand immIOffset2()
4187 %{
4188 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4189 match(ConI);
4190
4191 op_cost(0);
4192 format %{ %}
4193 interface(CONST_INTER);
4194 %}
4195
4196 operand immIOffset4()
4197 %{
4198 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4199 match(ConI);
4200
4201 op_cost(0);
4202 format %{ %}
4203 interface(CONST_INTER);
4204 %}
4205
4206 operand immIOffset8()
4207 %{
4208 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4209 match(ConI);
4210
4211 op_cost(0);
4212 format %{ %}
4213 interface(CONST_INTER);
4214 %}
4215
4216 operand immIOffset16()
4217 %{
4218 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4219 match(ConI);
4220
4221 op_cost(0);
4222 format %{ %}
4223 interface(CONST_INTER);
4224 %}
4225
4226 operand immLOffset()
4227 %{
4228 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4229 match(ConL);
4230
4231 op_cost(0);
4232 format %{ %}
4233 interface(CONST_INTER);
4234 %}
4235
4236 operand immLoffset1()
4237 %{
4238 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4239 match(ConL);
4240
4241 op_cost(0);
4242 format %{ %}
4243 interface(CONST_INTER);
4244 %}
4245
4246 operand immLoffset2()
4247 %{
4248 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4249 match(ConL);
4250
4251 op_cost(0);
4252 format %{ %}
4253 interface(CONST_INTER);
4254 %}
4255
4256 operand immLoffset4()
4257 %{
4258 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4259 match(ConL);
4260
4261 op_cost(0);
4262 format %{ %}
4263 interface(CONST_INTER);
4264 %}
4265
4266 operand immLoffset8()
4267 %{
4268 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4269 match(ConL);
4270
4271 op_cost(0);
4272 format %{ %}
4273 interface(CONST_INTER);
4274 %}
4275
4276 operand immLoffset16()
4277 %{
4278 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4279 match(ConL);
4280
4281 op_cost(0);
4282 format %{ %}
4283 interface(CONST_INTER);
4284 %}
4285
4286 // 5 bit signed long integer
4287 operand immL5()
4288 %{
4289 predicate(Assembler::is_simm(n->get_long(), 5));
4290 match(ConL);
4291
4292 op_cost(0);
4293 format %{ %}
4294 interface(CONST_INTER);
4295 %}
4296
4297 // 7 bit unsigned long integer
4298 operand immLU7()
4299 %{
4300 predicate(Assembler::is_uimm(n->get_long(), 7));
4301 match(ConL);
4302
4303 op_cost(0);
4304 format %{ %}
4305 interface(CONST_INTER);
4306 %}
4307
4308 // 8 bit signed value.
4309 operand immI8()
4310 %{
4311 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4312 match(ConI);
4313
4314 op_cost(0);
4315 format %{ %}
4316 interface(CONST_INTER);
4317 %}
4318
4319 // 8 bit signed value (simm8), or #simm8 LSL 8.
4320 operand immIDupV()
4321 %{
4322 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4323 match(ConI);
4324
4325 op_cost(0);
4326 format %{ %}
4327 interface(CONST_INTER);
4328 %}
4329
4330 // 8 bit signed value (simm8), or #simm8 LSL 8.
4331 operand immLDupV()
4332 %{
4333 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4334 match(ConL);
4335
4336 op_cost(0);
4337 format %{ %}
4338 interface(CONST_INTER);
4339 %}
4340
4341 // 8 bit signed value (simm8), or #simm8 LSL 8.
4342 operand immHDupV()
4343 %{
4344 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4345 match(ConH);
4346
4347 op_cost(0);
4348 format %{ %}
4349 interface(CONST_INTER);
4350 %}
4351
4352 // 8 bit integer valid for vector add sub immediate
4353 operand immBAddSubV()
4354 %{
4355 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4356 match(ConI);
4357
4358 op_cost(0);
4359 format %{ %}
4360 interface(CONST_INTER);
4361 %}
4362
4363 // 32 bit integer valid for add sub immediate
4364 operand immIAddSub()
4365 %{
4366 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4367 match(ConI);
4368 op_cost(0);
4369 format %{ %}
4370 interface(CONST_INTER);
4371 %}
4372
4373 // 32 bit integer valid for vector add sub immediate
4374 operand immIAddSubV()
4375 %{
4376 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4377 match(ConI);
4378
4379 op_cost(0);
4380 format %{ %}
4381 interface(CONST_INTER);
4382 %}
4383
4384 // 32 bit unsigned integer valid for logical immediate
4385
4386 operand immBLog()
4387 %{
4388 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4389 match(ConI);
4390
4391 op_cost(0);
4392 format %{ %}
4393 interface(CONST_INTER);
4394 %}
4395
4396 operand immSLog()
4397 %{
4398 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4399 match(ConI);
4400
4401 op_cost(0);
4402 format %{ %}
4403 interface(CONST_INTER);
4404 %}
4405
4406 operand immILog()
4407 %{
4408 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4409 match(ConI);
4410
4411 op_cost(0);
4412 format %{ %}
4413 interface(CONST_INTER);
4414 %}
4415
4416 // Integer operands 64 bit
4417 // 64 bit immediate
4418 operand immL()
4419 %{
4420 match(ConL);
4421
4422 op_cost(0);
4423 format %{ %}
4424 interface(CONST_INTER);
4425 %}
4426
4427 // 64 bit zero
4428 operand immL0()
4429 %{
4430 predicate(n->get_long() == 0);
4431 match(ConL);
4432
4433 op_cost(0);
4434 format %{ %}
4435 interface(CONST_INTER);
4436 %}
4437
4438 // 64 bit unit decrement
4439 operand immL_M1()
4440 %{
4441 predicate(n->get_long() == -1);
4442 match(ConL);
4443
4444 op_cost(0);
4445 format %{ %}
4446 interface(CONST_INTER);
4447 %}
4448
4449 // 64 bit integer valid for add sub immediate
4450 operand immLAddSub()
4451 %{
4452 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4453 match(ConL);
4454 op_cost(0);
4455 format %{ %}
4456 interface(CONST_INTER);
4457 %}
4458
4459 // 64 bit integer valid for addv subv immediate
4460 operand immLAddSubV()
4461 %{
4462 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4463 match(ConL);
4464
4465 op_cost(0);
4466 format %{ %}
4467 interface(CONST_INTER);
4468 %}
4469
4470 // 64 bit integer valid for logical immediate
4471 operand immLLog()
4472 %{
4473 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4474 match(ConL);
4475 op_cost(0);
4476 format %{ %}
4477 interface(CONST_INTER);
4478 %}
4479
4480 // Long Immediate: low 32-bit mask
4481 operand immL_32bits()
4482 %{
4483 predicate(n->get_long() == 0xFFFFFFFFL);
4484 match(ConL);
4485 op_cost(0);
4486 format %{ %}
4487 interface(CONST_INTER);
4488 %}
4489
4490 // Pointer operands
4491 // Pointer Immediate
4492 operand immP()
4493 %{
4494 match(ConP);
4495
4496 op_cost(0);
4497 format %{ %}
4498 interface(CONST_INTER);
4499 %}
4500
4501 // nullptr Pointer Immediate
4502 operand immP0()
4503 %{
4504 predicate(n->get_ptr() == 0);
4505 match(ConP);
4506
4507 op_cost(0);
4508 format %{ %}
4509 interface(CONST_INTER);
4510 %}
4511
4512 // Pointer Immediate One
4513 // this is used in object initialization (initial object header)
4514 operand immP_1()
4515 %{
4516 predicate(n->get_ptr() == 1);
4517 match(ConP);
4518
4519 op_cost(0);
4520 format %{ %}
4521 interface(CONST_INTER);
4522 %}
4523
4524 // AOT Runtime Constants Address
4525 operand immAOTRuntimeConstantsAddress()
4526 %{
4527 // Check if the address is in the range of AOT Runtime Constants
4528 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4529 match(ConP);
4530
4531 op_cost(0);
4532 format %{ %}
4533 interface(CONST_INTER);
4534 %}
4535
4536 // Float and Double operands
4537 // Double Immediate
4538 operand immD()
4539 %{
4540 match(ConD);
4541 op_cost(0);
4542 format %{ %}
4543 interface(CONST_INTER);
4544 %}
4545
4546 // Double Immediate: +0.0d
4547 operand immD0()
4548 %{
4549 predicate(jlong_cast(n->getd()) == 0);
4550 match(ConD);
4551
4552 op_cost(0);
4553 format %{ %}
4554 interface(CONST_INTER);
4555 %}
4556
4557 // constant 'double +0.0'.
4558 operand immDPacked()
4559 %{
4560 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4561 match(ConD);
4562 op_cost(0);
4563 format %{ %}
4564 interface(CONST_INTER);
4565 %}
4566
4567 // Float Immediate
4568 operand immF()
4569 %{
4570 match(ConF);
4571 op_cost(0);
4572 format %{ %}
4573 interface(CONST_INTER);
4574 %}
4575
4576 // Float Immediate: +0.0f.
4577 operand immF0()
4578 %{
4579 predicate(jint_cast(n->getf()) == 0);
4580 match(ConF);
4581
4582 op_cost(0);
4583 format %{ %}
4584 interface(CONST_INTER);
4585 %}
4586
4587 // Half Float (FP16) Immediate
4588 operand immH()
4589 %{
4590 match(ConH);
4591 op_cost(0);
4592 format %{ %}
4593 interface(CONST_INTER);
4594 %}
4595
4596 //
4597 operand immFPacked()
4598 %{
4599 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4600 match(ConF);
4601 op_cost(0);
4602 format %{ %}
4603 interface(CONST_INTER);
4604 %}
4605
4606 // Narrow pointer operands
4607 // Narrow Pointer Immediate
4608 operand immN()
4609 %{
4610 match(ConN);
4611
4612 op_cost(0);
4613 format %{ %}
4614 interface(CONST_INTER);
4615 %}
4616
4617 // Narrow nullptr Pointer Immediate
4618 operand immN0()
4619 %{
4620 predicate(n->get_narrowcon() == 0);
4621 match(ConN);
4622
4623 op_cost(0);
4624 format %{ %}
4625 interface(CONST_INTER);
4626 %}
4627
4628 operand immNKlass()
4629 %{
4630 match(ConNKlass);
4631
4632 op_cost(0);
4633 format %{ %}
4634 interface(CONST_INTER);
4635 %}
4636
4637 // Integer 32 bit Register Operands
4638 // Integer 32 bitRegister (excludes SP)
4639 operand iRegI()
4640 %{
4641 constraint(ALLOC_IN_RC(any_reg32));
4642 match(RegI);
4643 match(iRegINoSp);
4644 op_cost(0);
4645 format %{ %}
4646 interface(REG_INTER);
4647 %}
4648
4649 // Integer 32 bit Register not Special
4650 operand iRegINoSp()
4651 %{
4652 constraint(ALLOC_IN_RC(no_special_reg32));
4653 match(RegI);
4654 op_cost(0);
4655 format %{ %}
4656 interface(REG_INTER);
4657 %}
4658
4659 // Integer 64 bit Register Operands
4660 // Integer 64 bit Register (includes SP)
4661 operand iRegL()
4662 %{
4663 constraint(ALLOC_IN_RC(any_reg));
4664 match(RegL);
4665 match(iRegLNoSp);
4666 op_cost(0);
4667 format %{ %}
4668 interface(REG_INTER);
4669 %}
4670
4671 // Integer 64 bit Register not Special
4672 operand iRegLNoSp()
4673 %{
4674 constraint(ALLOC_IN_RC(no_special_reg));
4675 match(RegL);
4676 match(iRegL_R0);
4677 format %{ %}
4678 interface(REG_INTER);
4679 %}
4680
4681 // Pointer Register Operands
4682 // Pointer Register
4683 operand iRegP()
4684 %{
4685 constraint(ALLOC_IN_RC(ptr_reg));
4686 match(RegP);
4687 match(iRegPNoSp);
4688 match(iRegP_R0);
4689 //match(iRegP_R2);
4690 //match(iRegP_R4);
4691 match(iRegP_R5);
4692 match(thread_RegP);
4693 op_cost(0);
4694 format %{ %}
4695 interface(REG_INTER);
4696 %}
4697
4698 // Pointer 64 bit Register not Special
4699 operand iRegPNoSp()
4700 %{
4701 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4702 match(RegP);
4703 // match(iRegP);
4704 // match(iRegP_R0);
4705 // match(iRegP_R2);
4706 // match(iRegP_R4);
4707 // match(iRegP_R5);
4708 // match(thread_RegP);
4709 op_cost(0);
4710 format %{ %}
4711 interface(REG_INTER);
4712 %}
4713
4714 // This operand is not allowed to use rfp even if
4715 // rfp is not used to hold the frame pointer.
4716 operand iRegPNoSpNoRfp()
4717 %{
4718 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4719 match(RegP);
4720 match(iRegPNoSp);
4721 op_cost(0);
4722 format %{ %}
4723 interface(REG_INTER);
4724 %}
4725
4726 // Pointer 64 bit Register R0 only
4727 operand iRegP_R0()
4728 %{
4729 constraint(ALLOC_IN_RC(r0_reg));
4730 match(RegP);
4731 // match(iRegP);
4732 match(iRegPNoSp);
4733 op_cost(0);
4734 format %{ %}
4735 interface(REG_INTER);
4736 %}
4737
4738 // Pointer 64 bit Register R1 only
4739 operand iRegP_R1()
4740 %{
4741 constraint(ALLOC_IN_RC(r1_reg));
4742 match(RegP);
4743 // match(iRegP);
4744 match(iRegPNoSp);
4745 op_cost(0);
4746 format %{ %}
4747 interface(REG_INTER);
4748 %}
4749
4750 // Pointer 64 bit Register R2 only
4751 operand iRegP_R2()
4752 %{
4753 constraint(ALLOC_IN_RC(r2_reg));
4754 match(RegP);
4755 // match(iRegP);
4756 match(iRegPNoSp);
4757 op_cost(0);
4758 format %{ %}
4759 interface(REG_INTER);
4760 %}
4761
4762 // Pointer 64 bit Register R3 only
4763 operand iRegP_R3()
4764 %{
4765 constraint(ALLOC_IN_RC(r3_reg));
4766 match(RegP);
4767 // match(iRegP);
4768 match(iRegPNoSp);
4769 op_cost(0);
4770 format %{ %}
4771 interface(REG_INTER);
4772 %}
4773
4774 // Pointer 64 bit Register R4 only
4775 operand iRegP_R4()
4776 %{
4777 constraint(ALLOC_IN_RC(r4_reg));
4778 match(RegP);
4779 // match(iRegP);
4780 match(iRegPNoSp);
4781 op_cost(0);
4782 format %{ %}
4783 interface(REG_INTER);
4784 %}
4785
4786 // Pointer 64 bit Register R5 only
4787 operand iRegP_R5()
4788 %{
4789 constraint(ALLOC_IN_RC(r5_reg));
4790 match(RegP);
4791 // match(iRegP);
4792 match(iRegPNoSp);
4793 op_cost(0);
4794 format %{ %}
4795 interface(REG_INTER);
4796 %}
4797
4798 // Pointer 64 bit Register R10 only
4799 operand iRegP_R10()
4800 %{
4801 constraint(ALLOC_IN_RC(r10_reg));
4802 match(RegP);
4803 // match(iRegP);
4804 match(iRegPNoSp);
4805 op_cost(0);
4806 format %{ %}
4807 interface(REG_INTER);
4808 %}
4809
4810 // Long 64 bit Register R0 only
4811 operand iRegL_R0()
4812 %{
4813 constraint(ALLOC_IN_RC(r0_reg));
4814 match(RegL);
4815 match(iRegLNoSp);
4816 op_cost(0);
4817 format %{ %}
4818 interface(REG_INTER);
4819 %}
4820
4821 // Long 64 bit Register R11 only
4822 operand iRegL_R11()
4823 %{
4824 constraint(ALLOC_IN_RC(r11_reg));
4825 match(RegL);
4826 match(iRegLNoSp);
4827 op_cost(0);
4828 format %{ %}
4829 interface(REG_INTER);
4830 %}
4831
4832 // Register R0 only
4833 operand iRegI_R0()
4834 %{
4835 constraint(ALLOC_IN_RC(int_r0_reg));
4836 match(RegI);
4837 match(iRegINoSp);
4838 op_cost(0);
4839 format %{ %}
4840 interface(REG_INTER);
4841 %}
4842
4843 // Register R2 only
4844 operand iRegI_R2()
4845 %{
4846 constraint(ALLOC_IN_RC(int_r2_reg));
4847 match(RegI);
4848 match(iRegINoSp);
4849 op_cost(0);
4850 format %{ %}
4851 interface(REG_INTER);
4852 %}
4853
4854 // Register R3 only
4855 operand iRegI_R3()
4856 %{
4857 constraint(ALLOC_IN_RC(int_r3_reg));
4858 match(RegI);
4859 match(iRegINoSp);
4860 op_cost(0);
4861 format %{ %}
4862 interface(REG_INTER);
4863 %}
4864
4865
4866 // Register R4 only
4867 operand iRegI_R4()
4868 %{
4869 constraint(ALLOC_IN_RC(int_r4_reg));
4870 match(RegI);
4871 match(iRegINoSp);
4872 op_cost(0);
4873 format %{ %}
4874 interface(REG_INTER);
4875 %}
4876
4877
4878 // Pointer Register Operands
4879 // Narrow Pointer Register
4880 operand iRegN()
4881 %{
4882 constraint(ALLOC_IN_RC(any_reg32));
4883 match(RegN);
4884 match(iRegNNoSp);
4885 op_cost(0);
4886 format %{ %}
4887 interface(REG_INTER);
4888 %}
4889
4890 // Integer 64 bit Register not Special
4891 operand iRegNNoSp()
4892 %{
4893 constraint(ALLOC_IN_RC(no_special_reg32));
4894 match(RegN);
4895 op_cost(0);
4896 format %{ %}
4897 interface(REG_INTER);
4898 %}
4899
4900 // Float Register
4901 // Float register operands
4902 operand vRegF()
4903 %{
4904 constraint(ALLOC_IN_RC(float_reg));
4905 match(RegF);
4906
4907 op_cost(0);
4908 format %{ %}
4909 interface(REG_INTER);
4910 %}
4911
4912 // Double Register
4913 // Double register operands
4914 operand vRegD()
4915 %{
4916 constraint(ALLOC_IN_RC(double_reg));
4917 match(RegD);
4918
4919 op_cost(0);
4920 format %{ %}
4921 interface(REG_INTER);
4922 %}
4923
4924 // Generic vector class. This will be used for
4925 // all vector operands, including NEON and SVE.
4926 operand vReg()
4927 %{
4928 constraint(ALLOC_IN_RC(dynamic));
4929 match(VecA);
4930 match(VecD);
4931 match(VecX);
4932
4933 op_cost(0);
4934 format %{ %}
4935 interface(REG_INTER);
4936 %}
4937
4938 operand vReg_V10()
4939 %{
4940 constraint(ALLOC_IN_RC(v10_veca_reg));
4941 match(vReg);
4942
4943 op_cost(0);
4944 format %{ %}
4945 interface(REG_INTER);
4946 %}
4947
4948 operand vReg_V11()
4949 %{
4950 constraint(ALLOC_IN_RC(v11_veca_reg));
4951 match(vReg);
4952
4953 op_cost(0);
4954 format %{ %}
4955 interface(REG_INTER);
4956 %}
4957
4958 operand vReg_V12()
4959 %{
4960 constraint(ALLOC_IN_RC(v12_veca_reg));
4961 match(vReg);
4962
4963 op_cost(0);
4964 format %{ %}
4965 interface(REG_INTER);
4966 %}
4967
4968 operand vReg_V13()
4969 %{
4970 constraint(ALLOC_IN_RC(v13_veca_reg));
4971 match(vReg);
4972
4973 op_cost(0);
4974 format %{ %}
4975 interface(REG_INTER);
4976 %}
4977
4978 operand vReg_V17()
4979 %{
4980 constraint(ALLOC_IN_RC(v17_veca_reg));
4981 match(vReg);
4982
4983 op_cost(0);
4984 format %{ %}
4985 interface(REG_INTER);
4986 %}
4987
4988 operand vReg_V18()
4989 %{
4990 constraint(ALLOC_IN_RC(v18_veca_reg));
4991 match(vReg);
4992
4993 op_cost(0);
4994 format %{ %}
4995 interface(REG_INTER);
4996 %}
4997
4998 operand vReg_V23()
4999 %{
5000 constraint(ALLOC_IN_RC(v23_veca_reg));
5001 match(vReg);
5002
5003 op_cost(0);
5004 format %{ %}
5005 interface(REG_INTER);
5006 %}
5007
5008 operand vReg_V24()
5009 %{
5010 constraint(ALLOC_IN_RC(v24_veca_reg));
5011 match(vReg);
5012
5013 op_cost(0);
5014 format %{ %}
5015 interface(REG_INTER);
5016 %}
5017
5018 operand vecA()
5019 %{
5020 constraint(ALLOC_IN_RC(vectora_reg));
5021 match(VecA);
5022
5023 op_cost(0);
5024 format %{ %}
5025 interface(REG_INTER);
5026 %}
5027
5028 operand vecD()
5029 %{
5030 constraint(ALLOC_IN_RC(vectord_reg));
5031 match(VecD);
5032
5033 op_cost(0);
5034 format %{ %}
5035 interface(REG_INTER);
5036 %}
5037
5038 operand vecX()
5039 %{
5040 constraint(ALLOC_IN_RC(vectorx_reg));
5041 match(VecX);
5042
5043 op_cost(0);
5044 format %{ %}
5045 interface(REG_INTER);
5046 %}
5047
5048 operand vRegD_V0()
5049 %{
5050 constraint(ALLOC_IN_RC(v0_reg));
5051 match(RegD);
5052 op_cost(0);
5053 format %{ %}
5054 interface(REG_INTER);
5055 %}
5056
5057 operand vRegD_V1()
5058 %{
5059 constraint(ALLOC_IN_RC(v1_reg));
5060 match(RegD);
5061 op_cost(0);
5062 format %{ %}
5063 interface(REG_INTER);
5064 %}
5065
5066 operand vRegD_V2()
5067 %{
5068 constraint(ALLOC_IN_RC(v2_reg));
5069 match(RegD);
5070 op_cost(0);
5071 format %{ %}
5072 interface(REG_INTER);
5073 %}
5074
5075 operand vRegD_V3()
5076 %{
5077 constraint(ALLOC_IN_RC(v3_reg));
5078 match(RegD);
5079 op_cost(0);
5080 format %{ %}
5081 interface(REG_INTER);
5082 %}
5083
5084 operand vRegD_V4()
5085 %{
5086 constraint(ALLOC_IN_RC(v4_reg));
5087 match(RegD);
5088 op_cost(0);
5089 format %{ %}
5090 interface(REG_INTER);
5091 %}
5092
5093 operand vRegD_V5()
5094 %{
5095 constraint(ALLOC_IN_RC(v5_reg));
5096 match(RegD);
5097 op_cost(0);
5098 format %{ %}
5099 interface(REG_INTER);
5100 %}
5101
5102 operand vRegD_V6()
5103 %{
5104 constraint(ALLOC_IN_RC(v6_reg));
5105 match(RegD);
5106 op_cost(0);
5107 format %{ %}
5108 interface(REG_INTER);
5109 %}
5110
5111 operand vRegD_V7()
5112 %{
5113 constraint(ALLOC_IN_RC(v7_reg));
5114 match(RegD);
5115 op_cost(0);
5116 format %{ %}
5117 interface(REG_INTER);
5118 %}
5119
5120 operand vRegD_V12()
5121 %{
5122 constraint(ALLOC_IN_RC(v12_reg));
5123 match(RegD);
5124 op_cost(0);
5125 format %{ %}
5126 interface(REG_INTER);
5127 %}
5128
5129 operand vRegD_V13()
5130 %{
5131 constraint(ALLOC_IN_RC(v13_reg));
5132 match(RegD);
5133 op_cost(0);
5134 format %{ %}
5135 interface(REG_INTER);
5136 %}
5137
5138 operand pReg()
5139 %{
5140 constraint(ALLOC_IN_RC(pr_reg));
5141 match(RegVectMask);
5142 match(pRegGov);
5143 op_cost(0);
5144 format %{ %}
5145 interface(REG_INTER);
5146 %}
5147
5148 operand pRegGov()
5149 %{
5150 constraint(ALLOC_IN_RC(gov_pr));
5151 match(RegVectMask);
5152 match(pReg);
5153 op_cost(0);
5154 format %{ %}
5155 interface(REG_INTER);
5156 %}
5157
5158 operand pRegGov_P0()
5159 %{
5160 constraint(ALLOC_IN_RC(p0_reg));
5161 match(RegVectMask);
5162 op_cost(0);
5163 format %{ %}
5164 interface(REG_INTER);
5165 %}
5166
5167 operand pRegGov_P1()
5168 %{
5169 constraint(ALLOC_IN_RC(p1_reg));
5170 match(RegVectMask);
5171 op_cost(0);
5172 format %{ %}
5173 interface(REG_INTER);
5174 %}
5175
5176 // Flags register, used as output of signed compare instructions
5177
5178 // note that on AArch64 we also use this register as the output for
5179 // for floating point compare instructions (CmpF CmpD). this ensures
5180 // that ordered inequality tests use GT, GE, LT or LE none of which
5181 // pass through cases where the result is unordered i.e. one or both
5182 // inputs to the compare is a NaN. this means that the ideal code can
5183 // replace e.g. a GT with an LE and not end up capturing the NaN case
5184 // (where the comparison should always fail). EQ and NE tests are
5185 // always generated in ideal code so that unordered folds into the NE
5186 // case, matching the behaviour of AArch64 NE.
5187 //
5188 // This differs from x86 where the outputs of FP compares use a
5189 // special FP flags registers and where compares based on this
5190 // register are distinguished into ordered inequalities (cmpOpUCF) and
5191 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5192 // to explicitly handle the unordered case in branches. x86 also has
5193 // to include extra CMoveX rules to accept a cmpOpUCF input.
5194
5195 operand rFlagsReg()
5196 %{
5197 constraint(ALLOC_IN_RC(int_flags));
5198 match(RegFlags);
5199
5200 op_cost(0);
5201 format %{ "RFLAGS" %}
5202 interface(REG_INTER);
5203 %}
5204
5205 // Flags register, used as output of unsigned compare instructions
5206 operand rFlagsRegU()
5207 %{
5208 constraint(ALLOC_IN_RC(int_flags));
5209 match(RegFlags);
5210
5211 op_cost(0);
5212 format %{ "RFLAGSU" %}
5213 interface(REG_INTER);
5214 %}
5215
5216 // Special Registers
5217
5218 // Method Register
5219 operand inline_cache_RegP(iRegP reg)
5220 %{
5221 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5222 match(reg);
5223 match(iRegPNoSp);
5224 op_cost(0);
5225 format %{ %}
5226 interface(REG_INTER);
5227 %}
5228
5229 // Thread Register
5230 operand thread_RegP(iRegP reg)
5231 %{
5232 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5233 match(reg);
5234 op_cost(0);
5235 format %{ %}
5236 interface(REG_INTER);
5237 %}
5238
5239 //----------Memory Operands----------------------------------------------------
5240
5241 operand indirect(iRegP reg)
5242 %{
5243 constraint(ALLOC_IN_RC(ptr_reg));
5244 match(reg);
5245 op_cost(0);
5246 format %{ "[$reg]" %}
5247 interface(MEMORY_INTER) %{
5248 base($reg);
5249 index(0xffffffff);
5250 scale(0x0);
5251 disp(0x0);
5252 %}
5253 %}
5254
5255 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5256 %{
5257 constraint(ALLOC_IN_RC(ptr_reg));
5258 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5259 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5260 op_cost(0);
5261 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5262 interface(MEMORY_INTER) %{
5263 base($reg);
5264 index($ireg);
5265 scale($scale);
5266 disp(0x0);
5267 %}
5268 %}
5269
5270 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5271 %{
5272 constraint(ALLOC_IN_RC(ptr_reg));
5273 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5274 match(AddP reg (LShiftL lreg scale));
5275 op_cost(0);
5276 format %{ "$reg, $lreg lsl($scale)" %}
5277 interface(MEMORY_INTER) %{
5278 base($reg);
5279 index($lreg);
5280 scale($scale);
5281 disp(0x0);
5282 %}
5283 %}
5284
5285 operand indIndexI2L(iRegP reg, iRegI ireg)
5286 %{
5287 constraint(ALLOC_IN_RC(ptr_reg));
5288 match(AddP reg (ConvI2L ireg));
5289 op_cost(0);
5290 format %{ "$reg, $ireg, 0, I2L" %}
5291 interface(MEMORY_INTER) %{
5292 base($reg);
5293 index($ireg);
5294 scale(0x0);
5295 disp(0x0);
5296 %}
5297 %}
5298
5299 operand indIndex(iRegP reg, iRegL lreg)
5300 %{
5301 constraint(ALLOC_IN_RC(ptr_reg));
5302 match(AddP reg lreg);
5303 op_cost(0);
5304 format %{ "$reg, $lreg" %}
5305 interface(MEMORY_INTER) %{
5306 base($reg);
5307 index($lreg);
5308 scale(0x0);
5309 disp(0x0);
5310 %}
5311 %}
5312
5313 operand indOffI1(iRegP reg, immIOffset1 off)
5314 %{
5315 constraint(ALLOC_IN_RC(ptr_reg));
5316 match(AddP reg off);
5317 op_cost(0);
5318 format %{ "[$reg, $off]" %}
5319 interface(MEMORY_INTER) %{
5320 base($reg);
5321 index(0xffffffff);
5322 scale(0x0);
5323 disp($off);
5324 %}
5325 %}
5326
5327 operand indOffI2(iRegP reg, immIOffset2 off)
5328 %{
5329 constraint(ALLOC_IN_RC(ptr_reg));
5330 match(AddP reg off);
5331 op_cost(0);
5332 format %{ "[$reg, $off]" %}
5333 interface(MEMORY_INTER) %{
5334 base($reg);
5335 index(0xffffffff);
5336 scale(0x0);
5337 disp($off);
5338 %}
5339 %}
5340
5341 operand indOffI4(iRegP reg, immIOffset4 off)
5342 %{
5343 constraint(ALLOC_IN_RC(ptr_reg));
5344 match(AddP reg off);
5345 op_cost(0);
5346 format %{ "[$reg, $off]" %}
5347 interface(MEMORY_INTER) %{
5348 base($reg);
5349 index(0xffffffff);
5350 scale(0x0);
5351 disp($off);
5352 %}
5353 %}
5354
5355 operand indOffI8(iRegP reg, immIOffset8 off)
5356 %{
5357 constraint(ALLOC_IN_RC(ptr_reg));
5358 match(AddP reg off);
5359 op_cost(0);
5360 format %{ "[$reg, $off]" %}
5361 interface(MEMORY_INTER) %{
5362 base($reg);
5363 index(0xffffffff);
5364 scale(0x0);
5365 disp($off);
5366 %}
5367 %}
5368
5369 operand indOffI16(iRegP reg, immIOffset16 off)
5370 %{
5371 constraint(ALLOC_IN_RC(ptr_reg));
5372 match(AddP reg off);
5373 op_cost(0);
5374 format %{ "[$reg, $off]" %}
5375 interface(MEMORY_INTER) %{
5376 base($reg);
5377 index(0xffffffff);
5378 scale(0x0);
5379 disp($off);
5380 %}
5381 %}
5382
5383 operand indOffL1(iRegP reg, immLoffset1 off)
5384 %{
5385 constraint(ALLOC_IN_RC(ptr_reg));
5386 match(AddP reg off);
5387 op_cost(0);
5388 format %{ "[$reg, $off]" %}
5389 interface(MEMORY_INTER) %{
5390 base($reg);
5391 index(0xffffffff);
5392 scale(0x0);
5393 disp($off);
5394 %}
5395 %}
5396
5397 operand indOffL2(iRegP reg, immLoffset2 off)
5398 %{
5399 constraint(ALLOC_IN_RC(ptr_reg));
5400 match(AddP reg off);
5401 op_cost(0);
5402 format %{ "[$reg, $off]" %}
5403 interface(MEMORY_INTER) %{
5404 base($reg);
5405 index(0xffffffff);
5406 scale(0x0);
5407 disp($off);
5408 %}
5409 %}
5410
5411 operand indOffL4(iRegP reg, immLoffset4 off)
5412 %{
5413 constraint(ALLOC_IN_RC(ptr_reg));
5414 match(AddP reg off);
5415 op_cost(0);
5416 format %{ "[$reg, $off]" %}
5417 interface(MEMORY_INTER) %{
5418 base($reg);
5419 index(0xffffffff);
5420 scale(0x0);
5421 disp($off);
5422 %}
5423 %}
5424
5425 operand indOffL8(iRegP reg, immLoffset8 off)
5426 %{
5427 constraint(ALLOC_IN_RC(ptr_reg));
5428 match(AddP reg off);
5429 op_cost(0);
5430 format %{ "[$reg, $off]" %}
5431 interface(MEMORY_INTER) %{
5432 base($reg);
5433 index(0xffffffff);
5434 scale(0x0);
5435 disp($off);
5436 %}
5437 %}
5438
5439 operand indOffL16(iRegP reg, immLoffset16 off)
5440 %{
5441 constraint(ALLOC_IN_RC(ptr_reg));
5442 match(AddP reg off);
5443 op_cost(0);
5444 format %{ "[$reg, $off]" %}
5445 interface(MEMORY_INTER) %{
5446 base($reg);
5447 index(0xffffffff);
5448 scale(0x0);
5449 disp($off);
5450 %}
5451 %}
5452
5453 operand indirectX2P(iRegL reg)
5454 %{
5455 constraint(ALLOC_IN_RC(ptr_reg));
5456 match(CastX2P reg);
5457 op_cost(0);
5458 format %{ "[$reg]\t# long -> ptr" %}
5459 interface(MEMORY_INTER) %{
5460 base($reg);
5461 index(0xffffffff);
5462 scale(0x0);
5463 disp(0x0);
5464 %}
5465 %}
5466
5467 operand indOffX2P(iRegL reg, immLOffset off)
5468 %{
5469 constraint(ALLOC_IN_RC(ptr_reg));
5470 match(AddP (CastX2P reg) off);
5471 op_cost(0);
5472 format %{ "[$reg, $off]\t# long -> ptr" %}
5473 interface(MEMORY_INTER) %{
5474 base($reg);
5475 index(0xffffffff);
5476 scale(0x0);
5477 disp($off);
5478 %}
5479 %}
5480
5481 operand indirectN(iRegN reg)
5482 %{
5483 predicate(CompressedOops::shift() == 0);
5484 constraint(ALLOC_IN_RC(ptr_reg));
5485 match(DecodeN reg);
5486 op_cost(0);
5487 format %{ "[$reg]\t# narrow" %}
5488 interface(MEMORY_INTER) %{
5489 base($reg);
5490 index(0xffffffff);
5491 scale(0x0);
5492 disp(0x0);
5493 %}
5494 %}
5495
5496 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5497 %{
5498 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5499 constraint(ALLOC_IN_RC(ptr_reg));
5500 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5501 op_cost(0);
5502 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5503 interface(MEMORY_INTER) %{
5504 base($reg);
5505 index($ireg);
5506 scale($scale);
5507 disp(0x0);
5508 %}
5509 %}
5510
5511 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5512 %{
5513 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5514 constraint(ALLOC_IN_RC(ptr_reg));
5515 match(AddP (DecodeN reg) (LShiftL lreg scale));
5516 op_cost(0);
5517 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5518 interface(MEMORY_INTER) %{
5519 base($reg);
5520 index($lreg);
5521 scale($scale);
5522 disp(0x0);
5523 %}
5524 %}
5525
5526 operand indIndexI2LN(iRegN reg, iRegI ireg)
5527 %{
5528 predicate(CompressedOops::shift() == 0);
5529 constraint(ALLOC_IN_RC(ptr_reg));
5530 match(AddP (DecodeN reg) (ConvI2L ireg));
5531 op_cost(0);
5532 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5533 interface(MEMORY_INTER) %{
5534 base($reg);
5535 index($ireg);
5536 scale(0x0);
5537 disp(0x0);
5538 %}
5539 %}
5540
5541 operand indIndexN(iRegN reg, iRegL lreg)
5542 %{
5543 predicate(CompressedOops::shift() == 0);
5544 constraint(ALLOC_IN_RC(ptr_reg));
5545 match(AddP (DecodeN reg) lreg);
5546 op_cost(0);
5547 format %{ "$reg, $lreg\t# narrow" %}
5548 interface(MEMORY_INTER) %{
5549 base($reg);
5550 index($lreg);
5551 scale(0x0);
5552 disp(0x0);
5553 %}
5554 %}
5555
5556 operand indOffIN(iRegN reg, immIOffset off)
5557 %{
5558 predicate(CompressedOops::shift() == 0);
5559 constraint(ALLOC_IN_RC(ptr_reg));
5560 match(AddP (DecodeN reg) off);
5561 op_cost(0);
5562 format %{ "[$reg, $off]\t# narrow" %}
5563 interface(MEMORY_INTER) %{
5564 base($reg);
5565 index(0xffffffff);
5566 scale(0x0);
5567 disp($off);
5568 %}
5569 %}
5570
5571 operand indOffLN(iRegN reg, immLOffset off)
5572 %{
5573 predicate(CompressedOops::shift() == 0);
5574 constraint(ALLOC_IN_RC(ptr_reg));
5575 match(AddP (DecodeN reg) off);
5576 op_cost(0);
5577 format %{ "[$reg, $off]\t# narrow" %}
5578 interface(MEMORY_INTER) %{
5579 base($reg);
5580 index(0xffffffff);
5581 scale(0x0);
5582 disp($off);
5583 %}
5584 %}
5585
5586
5587 //----------Special Memory Operands--------------------------------------------
5588 // Stack Slot Operand - This operand is used for loading and storing temporary
5589 // values on the stack where a match requires a value to
5590 // flow through memory.
5591 operand stackSlotP(sRegP reg)
5592 %{
5593 constraint(ALLOC_IN_RC(stack_slots));
5594 op_cost(100);
5595 // No match rule because this operand is only generated in matching
5596 // match(RegP);
5597 format %{ "[$reg]" %}
5598 interface(MEMORY_INTER) %{
5599 base(0x1e); // RSP
5600 index(0x0); // No Index
5601 scale(0x0); // No Scale
5602 disp($reg); // Stack Offset
5603 %}
5604 %}
5605
5606 operand stackSlotI(sRegI reg)
5607 %{
5608 constraint(ALLOC_IN_RC(stack_slots));
5609 // No match rule because this operand is only generated in matching
5610 // match(RegI);
5611 format %{ "[$reg]" %}
5612 interface(MEMORY_INTER) %{
5613 base(0x1e); // RSP
5614 index(0x0); // No Index
5615 scale(0x0); // No Scale
5616 disp($reg); // Stack Offset
5617 %}
5618 %}
5619
5620 operand stackSlotF(sRegF reg)
5621 %{
5622 constraint(ALLOC_IN_RC(stack_slots));
5623 // No match rule because this operand is only generated in matching
5624 // match(RegF);
5625 format %{ "[$reg]" %}
5626 interface(MEMORY_INTER) %{
5627 base(0x1e); // RSP
5628 index(0x0); // No Index
5629 scale(0x0); // No Scale
5630 disp($reg); // Stack Offset
5631 %}
5632 %}
5633
5634 operand stackSlotD(sRegD reg)
5635 %{
5636 constraint(ALLOC_IN_RC(stack_slots));
5637 // No match rule because this operand is only generated in matching
5638 // match(RegD);
5639 format %{ "[$reg]" %}
5640 interface(MEMORY_INTER) %{
5641 base(0x1e); // RSP
5642 index(0x0); // No Index
5643 scale(0x0); // No Scale
5644 disp($reg); // Stack Offset
5645 %}
5646 %}
5647
5648 operand stackSlotL(sRegL reg)
5649 %{
5650 constraint(ALLOC_IN_RC(stack_slots));
5651 // No match rule because this operand is only generated in matching
5652 // match(RegL);
5653 format %{ "[$reg]" %}
5654 interface(MEMORY_INTER) %{
5655 base(0x1e); // RSP
5656 index(0x0); // No Index
5657 scale(0x0); // No Scale
5658 disp($reg); // Stack Offset
5659 %}
5660 %}
5661
5662 // Operands for expressing Control Flow
5663 // NOTE: Label is a predefined operand which should not be redefined in
5664 // the AD file. It is generically handled within the ADLC.
5665
5666 //----------Conditional Branch Operands----------------------------------------
5667 // Comparison Op - This is the operation of the comparison, and is limited to
5668 // the following set of codes:
5669 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5670 //
5671 // Other attributes of the comparison, such as unsignedness, are specified
5672 // by the comparison instruction that sets a condition code flags register.
5673 // That result is represented by a flags operand whose subtype is appropriate
5674 // to the unsignedness (etc.) of the comparison.
5675 //
5676 // Later, the instruction which matches both the Comparison Op (a Bool) and
5677 // the flags (produced by the Cmp) specifies the coding of the comparison op
5678 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5679
5680 // used for signed integral comparisons and fp comparisons
5681
5682 operand cmpOp()
5683 %{
5684 match(Bool);
5685
5686 format %{ "" %}
5687 interface(COND_INTER) %{
5688 equal(0x0, "eq");
5689 not_equal(0x1, "ne");
5690 less(0xb, "lt");
5691 greater_equal(0xa, "ge");
5692 less_equal(0xd, "le");
5693 greater(0xc, "gt");
5694 overflow(0x6, "vs");
5695 no_overflow(0x7, "vc");
5696 %}
5697 %}
5698
5699 // used for unsigned integral comparisons
5700
5701 operand cmpOpU()
5702 %{
5703 match(Bool);
5704
5705 format %{ "" %}
5706 interface(COND_INTER) %{
5707 equal(0x0, "eq");
5708 not_equal(0x1, "ne");
5709 less(0x3, "lo");
5710 greater_equal(0x2, "hs");
5711 less_equal(0x9, "ls");
5712 greater(0x8, "hi");
5713 overflow(0x6, "vs");
5714 no_overflow(0x7, "vc");
5715 %}
5716 %}
5717
5718 // used for certain integral comparisons which can be
5719 // converted to cbxx or tbxx instructions
5720
5721 operand cmpOpEqNe()
5722 %{
5723 match(Bool);
5724 op_cost(0);
5725 predicate(n->as_Bool()->_test._test == BoolTest::ne
5726 || n->as_Bool()->_test._test == BoolTest::eq);
5727
5728 format %{ "" %}
5729 interface(COND_INTER) %{
5730 equal(0x0, "eq");
5731 not_equal(0x1, "ne");
5732 less(0xb, "lt");
5733 greater_equal(0xa, "ge");
5734 less_equal(0xd, "le");
5735 greater(0xc, "gt");
5736 overflow(0x6, "vs");
5737 no_overflow(0x7, "vc");
5738 %}
5739 %}
5740
5741 // used for certain integral comparisons which can be
5742 // converted to cbxx or tbxx instructions
5743
5744 operand cmpOpLtGe()
5745 %{
5746 match(Bool);
5747 op_cost(0);
5748
5749 predicate(n->as_Bool()->_test._test == BoolTest::lt
5750 || n->as_Bool()->_test._test == BoolTest::ge);
5751
5752 format %{ "" %}
5753 interface(COND_INTER) %{
5754 equal(0x0, "eq");
5755 not_equal(0x1, "ne");
5756 less(0xb, "lt");
5757 greater_equal(0xa, "ge");
5758 less_equal(0xd, "le");
5759 greater(0xc, "gt");
5760 overflow(0x6, "vs");
5761 no_overflow(0x7, "vc");
5762 %}
5763 %}
5764
5765 // used for certain unsigned integral comparisons which can be
5766 // converted to cbxx or tbxx instructions
5767
5768 operand cmpOpUEqNeLeGt()
5769 %{
5770 match(Bool);
5771 op_cost(0);
5772
5773 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5774 n->as_Bool()->_test._test == BoolTest::ne ||
5775 n->as_Bool()->_test._test == BoolTest::le ||
5776 n->as_Bool()->_test._test == BoolTest::gt);
5777
5778 format %{ "" %}
5779 interface(COND_INTER) %{
5780 equal(0x0, "eq");
5781 not_equal(0x1, "ne");
5782 less(0x3, "lo");
5783 greater_equal(0x2, "hs");
5784 less_equal(0x9, "ls");
5785 greater(0x8, "hi");
5786 overflow(0x6, "vs");
5787 no_overflow(0x7, "vc");
5788 %}
5789 %}
5790
5791 // Special operand allowing long args to int ops to be truncated for free
5792
5793 operand iRegL2I(iRegL reg) %{
5794
5795 op_cost(0);
5796
5797 match(ConvL2I reg);
5798
5799 format %{ "l2i($reg)" %}
5800
5801 interface(REG_INTER)
5802 %}
5803
5804 operand iRegL2P(iRegL reg) %{
5805
5806 op_cost(0);
5807
5808 match(CastX2P reg);
5809
5810 format %{ "l2p($reg)" %}
5811
5812 interface(REG_INTER)
5813 %}
5814
5815 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5816 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5817 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5818 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5819
5820 //----------OPERAND CLASSES----------------------------------------------------
5821 // Operand Classes are groups of operands that are used as to simplify
5822 // instruction definitions by not requiring the AD writer to specify
5823 // separate instructions for every form of operand when the
5824 // instruction accepts multiple operand types with the same basic
5825 // encoding and format. The classic case of this is memory operands.
5826
5827 // memory is used to define read/write location for load/store
5828 // instruction defs. we can turn a memory op into an Address
5829
5830 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5831 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5832
5833 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5834 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5835
5836 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5837 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5838
5839 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5840 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5841
5842 // All of the memory operands. For the pipeline description.
5843 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5844 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5845 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5846
5847
5848 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5849 // operations. it allows the src to be either an iRegI or a (ConvL2I
5850 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5851 // can be elided because the 32-bit instruction will just employ the
5852 // lower 32 bits anyway.
5853 //
5854 // n.b. this does not elide all L2I conversions. if the truncated
5855 // value is consumed by more than one operation then the ConvL2I
5856 // cannot be bundled into the consuming nodes so an l2i gets planted
5857 // (actually a movw $dst $src) and the downstream instructions consume
5858 // the result of the l2i as an iRegI input. That's a shame since the
5859 // movw is actually redundant but its not too costly.
5860
5861 opclass iRegIorL2I(iRegI, iRegL2I);
5862 opclass iRegPorL2P(iRegP, iRegL2P);
5863
5864 //----------PIPELINE-----------------------------------------------------------
5865 // Rules which define the behavior of the target architectures pipeline.
5866
5867 // For specific pipelines, eg A53, define the stages of that pipeline
5868 //pipe_desc(ISS, EX1, EX2, WR);
5869 #define ISS S0
5870 #define EX1 S1
5871 #define EX2 S2
5872 #define WR S3
5873
5874 // Integer ALU reg operation
5875 pipeline %{
5876
5877 attributes %{
5878 // ARM instructions are of fixed length
5879 fixed_size_instructions; // Fixed size instructions TODO does
5880 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5881 // ARM instructions come in 32-bit word units
5882 instruction_unit_size = 4; // An instruction is 4 bytes long
5883 instruction_fetch_unit_size = 64; // The processor fetches one line
5884 instruction_fetch_units = 1; // of 64 bytes
5885 %}
5886
5887 // We don't use an actual pipeline model so don't care about resources
5888 // or description. we do use pipeline classes to introduce fixed
5889 // latencies
5890
5891 //----------RESOURCES----------------------------------------------------------
5892 // Resources are the functional units available to the machine
5893
5894 resources( INS0, INS1, INS01 = INS0 | INS1,
5895 ALU0, ALU1, ALU = ALU0 | ALU1,
5896 MAC,
5897 DIV,
5898 BRANCH,
5899 LDST,
5900 NEON_FP);
5901
5902 //----------PIPELINE DESCRIPTION-----------------------------------------------
5903 // Pipeline Description specifies the stages in the machine's pipeline
5904
5905 // Define the pipeline as a generic 6 stage pipeline
5906 pipe_desc(S0, S1, S2, S3, S4, S5);
5907
5908 //----------PIPELINE CLASSES---------------------------------------------------
5909 // Pipeline Classes describe the stages in which input and output are
5910 // referenced by the hardware pipeline.
5911
5912 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5913 %{
5914 single_instruction;
5915 src1 : S1(read);
5916 src2 : S2(read);
5917 dst : S5(write);
5918 INS01 : ISS;
5919 NEON_FP : S5;
5920 %}
5921
5922 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5923 %{
5924 single_instruction;
5925 src1 : S1(read);
5926 src2 : S2(read);
5927 dst : S5(write);
5928 INS01 : ISS;
5929 NEON_FP : S5;
5930 %}
5931
5932 pipe_class fp_uop_s(vRegF dst, vRegF src)
5933 %{
5934 single_instruction;
5935 src : S1(read);
5936 dst : S5(write);
5937 INS01 : ISS;
5938 NEON_FP : S5;
5939 %}
5940
5941 pipe_class fp_uop_d(vRegD dst, vRegD src)
5942 %{
5943 single_instruction;
5944 src : S1(read);
5945 dst : S5(write);
5946 INS01 : ISS;
5947 NEON_FP : S5;
5948 %}
5949
5950 pipe_class fp_d2f(vRegF dst, vRegD src)
5951 %{
5952 single_instruction;
5953 src : S1(read);
5954 dst : S5(write);
5955 INS01 : ISS;
5956 NEON_FP : S5;
5957 %}
5958
5959 pipe_class fp_f2d(vRegD dst, vRegF src)
5960 %{
5961 single_instruction;
5962 src : S1(read);
5963 dst : S5(write);
5964 INS01 : ISS;
5965 NEON_FP : S5;
5966 %}
5967
5968 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5969 %{
5970 single_instruction;
5971 src : S1(read);
5972 dst : S5(write);
5973 INS01 : ISS;
5974 NEON_FP : S5;
5975 %}
5976
5977 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5978 %{
5979 single_instruction;
5980 src : S1(read);
5981 dst : S5(write);
5982 INS01 : ISS;
5983 NEON_FP : S5;
5984 %}
5985
5986 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5987 %{
5988 single_instruction;
5989 src : S1(read);
5990 dst : S5(write);
5991 INS01 : ISS;
5992 NEON_FP : S5;
5993 %}
5994
5995 pipe_class fp_l2f(vRegF dst, iRegL src)
5996 %{
5997 single_instruction;
5998 src : S1(read);
5999 dst : S5(write);
6000 INS01 : ISS;
6001 NEON_FP : S5;
6002 %}
6003
6004 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6005 %{
6006 single_instruction;
6007 src : S1(read);
6008 dst : S5(write);
6009 INS01 : ISS;
6010 NEON_FP : S5;
6011 %}
6012
6013 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6014 %{
6015 single_instruction;
6016 src : S1(read);
6017 dst : S5(write);
6018 INS01 : ISS;
6019 NEON_FP : S5;
6020 %}
6021
6022 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6023 %{
6024 single_instruction;
6025 src : S1(read);
6026 dst : S5(write);
6027 INS01 : ISS;
6028 NEON_FP : S5;
6029 %}
6030
6031 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6032 %{
6033 single_instruction;
6034 src : S1(read);
6035 dst : S5(write);
6036 INS01 : ISS;
6037 NEON_FP : S5;
6038 %}
6039
6040 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6041 %{
6042 single_instruction;
6043 src1 : S1(read);
6044 src2 : S2(read);
6045 dst : S5(write);
6046 INS0 : ISS;
6047 NEON_FP : S5;
6048 %}
6049
6050 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6051 %{
6052 single_instruction;
6053 src1 : S1(read);
6054 src2 : S2(read);
6055 dst : S5(write);
6056 INS0 : ISS;
6057 NEON_FP : S5;
6058 %}
6059
6060 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6061 %{
6062 single_instruction;
6063 cr : S1(read);
6064 src1 : S1(read);
6065 src2 : S1(read);
6066 dst : S3(write);
6067 INS01 : ISS;
6068 NEON_FP : S3;
6069 %}
6070
6071 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6072 %{
6073 single_instruction;
6074 cr : S1(read);
6075 src1 : S1(read);
6076 src2 : S1(read);
6077 dst : S3(write);
6078 INS01 : ISS;
6079 NEON_FP : S3;
6080 %}
6081
6082 pipe_class fp_imm_s(vRegF dst)
6083 %{
6084 single_instruction;
6085 dst : S3(write);
6086 INS01 : ISS;
6087 NEON_FP : S3;
6088 %}
6089
6090 pipe_class fp_imm_d(vRegD dst)
6091 %{
6092 single_instruction;
6093 dst : S3(write);
6094 INS01 : ISS;
6095 NEON_FP : S3;
6096 %}
6097
6098 pipe_class fp_load_constant_s(vRegF dst)
6099 %{
6100 single_instruction;
6101 dst : S4(write);
6102 INS01 : ISS;
6103 NEON_FP : S4;
6104 %}
6105
6106 pipe_class fp_load_constant_d(vRegD dst)
6107 %{
6108 single_instruction;
6109 dst : S4(write);
6110 INS01 : ISS;
6111 NEON_FP : S4;
6112 %}
6113
6114 //------- Integer ALU operations --------------------------
6115
6116 // Integer ALU reg-reg operation
6117 // Operands needed in EX1, result generated in EX2
6118 // Eg. ADD x0, x1, x2
6119 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6120 %{
6121 single_instruction;
6122 dst : EX2(write);
6123 src1 : EX1(read);
6124 src2 : EX1(read);
6125 INS01 : ISS; // Dual issue as instruction 0 or 1
6126 ALU : EX2;
6127 %}
6128
6129 // Integer ALU reg-reg operation with constant shift
6130 // Shifted register must be available in LATE_ISS instead of EX1
6131 // Eg. ADD x0, x1, x2, LSL #2
6132 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6133 %{
6134 single_instruction;
6135 dst : EX2(write);
6136 src1 : EX1(read);
6137 src2 : ISS(read);
6138 INS01 : ISS;
6139 ALU : EX2;
6140 %}
6141
6142 // Integer ALU reg operation with constant shift
6143 // Eg. LSL x0, x1, #shift
6144 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6145 %{
6146 single_instruction;
6147 dst : EX2(write);
6148 src1 : ISS(read);
6149 INS01 : ISS;
6150 ALU : EX2;
6151 %}
6152
6153 // Integer ALU reg-reg operation with variable shift
6154 // Both operands must be available in LATE_ISS instead of EX1
6155 // Result is available in EX1 instead of EX2
6156 // Eg. LSLV x0, x1, x2
6157 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6158 %{
6159 single_instruction;
6160 dst : EX1(write);
6161 src1 : ISS(read);
6162 src2 : ISS(read);
6163 INS01 : ISS;
6164 ALU : EX1;
6165 %}
6166
6167 // Integer ALU reg-reg operation with extract
6168 // As for _vshift above, but result generated in EX2
6169 // Eg. EXTR x0, x1, x2, #N
6170 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6171 %{
6172 single_instruction;
6173 dst : EX2(write);
6174 src1 : ISS(read);
6175 src2 : ISS(read);
6176 INS1 : ISS; // Can only dual issue as Instruction 1
6177 ALU : EX1;
6178 %}
6179
6180 // Integer ALU reg operation
6181 // Eg. NEG x0, x1
6182 pipe_class ialu_reg(iRegI dst, iRegI src)
6183 %{
6184 single_instruction;
6185 dst : EX2(write);
6186 src : EX1(read);
6187 INS01 : ISS;
6188 ALU : EX2;
6189 %}
6190
6191 // Integer ALU reg mmediate operation
6192 // Eg. ADD x0, x1, #N
6193 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6194 %{
6195 single_instruction;
6196 dst : EX2(write);
6197 src1 : EX1(read);
6198 INS01 : ISS;
6199 ALU : EX2;
6200 %}
6201
6202 // Integer ALU immediate operation (no source operands)
6203 // Eg. MOV x0, #N
6204 pipe_class ialu_imm(iRegI dst)
6205 %{
6206 single_instruction;
6207 dst : EX1(write);
6208 INS01 : ISS;
6209 ALU : EX1;
6210 %}
6211
6212 //------- Compare operation -------------------------------
6213
6214 // Compare reg-reg
6215 // Eg. CMP x0, x1
6216 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6217 %{
6218 single_instruction;
6219 // fixed_latency(16);
6220 cr : EX2(write);
6221 op1 : EX1(read);
6222 op2 : EX1(read);
6223 INS01 : ISS;
6224 ALU : EX2;
6225 %}
6226
6227 // Compare reg-reg
6228 // Eg. CMP x0, #N
6229 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6230 %{
6231 single_instruction;
6232 // fixed_latency(16);
6233 cr : EX2(write);
6234 op1 : EX1(read);
6235 INS01 : ISS;
6236 ALU : EX2;
6237 %}
6238
6239 //------- Conditional instructions ------------------------
6240
6241 // Conditional no operands
6242 // Eg. CSINC x0, zr, zr, <cond>
6243 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6244 %{
6245 single_instruction;
6246 cr : EX1(read);
6247 dst : EX2(write);
6248 INS01 : ISS;
6249 ALU : EX2;
6250 %}
6251
6252 // Conditional 2 operand
6253 // EG. CSEL X0, X1, X2, <cond>
6254 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6255 %{
6256 single_instruction;
6257 cr : EX1(read);
6258 src1 : EX1(read);
6259 src2 : EX1(read);
6260 dst : EX2(write);
6261 INS01 : ISS;
6262 ALU : EX2;
6263 %}
6264
6265 // Conditional 2 operand
6266 // EG. CSEL X0, X1, X2, <cond>
6267 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6268 %{
6269 single_instruction;
6270 cr : EX1(read);
6271 src : EX1(read);
6272 dst : EX2(write);
6273 INS01 : ISS;
6274 ALU : EX2;
6275 %}
6276
6277 //------- Multiply pipeline operations --------------------
6278
6279 // Multiply reg-reg
6280 // Eg. MUL w0, w1, w2
6281 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6282 %{
6283 single_instruction;
6284 dst : WR(write);
6285 src1 : ISS(read);
6286 src2 : ISS(read);
6287 INS01 : ISS;
6288 MAC : WR;
6289 %}
6290
6291 // Multiply accumulate
6292 // Eg. MADD w0, w1, w2, w3
6293 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6294 %{
6295 single_instruction;
6296 dst : WR(write);
6297 src1 : ISS(read);
6298 src2 : ISS(read);
6299 src3 : ISS(read);
6300 INS01 : ISS;
6301 MAC : WR;
6302 %}
6303
6304 // Eg. MUL w0, w1, w2
6305 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6306 %{
6307 single_instruction;
6308 fixed_latency(3); // Maximum latency for 64 bit mul
6309 dst : WR(write);
6310 src1 : ISS(read);
6311 src2 : ISS(read);
6312 INS01 : ISS;
6313 MAC : WR;
6314 %}
6315
6316 // Multiply accumulate
6317 // Eg. MADD w0, w1, w2, w3
6318 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6319 %{
6320 single_instruction;
6321 fixed_latency(3); // Maximum latency for 64 bit mul
6322 dst : WR(write);
6323 src1 : ISS(read);
6324 src2 : ISS(read);
6325 src3 : ISS(read);
6326 INS01 : ISS;
6327 MAC : WR;
6328 %}
6329
6330 //------- Divide pipeline operations --------------------
6331
6332 // Eg. SDIV w0, w1, w2
6333 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6334 %{
6335 single_instruction;
6336 fixed_latency(8); // Maximum latency for 32 bit divide
6337 dst : WR(write);
6338 src1 : ISS(read);
6339 src2 : ISS(read);
6340 INS0 : ISS; // Can only dual issue as instruction 0
6341 DIV : WR;
6342 %}
6343
6344 // Eg. SDIV x0, x1, x2
6345 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6346 %{
6347 single_instruction;
6348 fixed_latency(16); // Maximum latency for 64 bit divide
6349 dst : WR(write);
6350 src1 : ISS(read);
6351 src2 : ISS(read);
6352 INS0 : ISS; // Can only dual issue as instruction 0
6353 DIV : WR;
6354 %}
6355
6356 //------- Load pipeline operations ------------------------
6357
6358 // Load - prefetch
6359 // Eg. PFRM <mem>
6360 pipe_class iload_prefetch(memory mem)
6361 %{
6362 single_instruction;
6363 mem : ISS(read);
6364 INS01 : ISS;
6365 LDST : WR;
6366 %}
6367
6368 // Load - reg, mem
6369 // Eg. LDR x0, <mem>
6370 pipe_class iload_reg_mem(iRegI dst, memory mem)
6371 %{
6372 single_instruction;
6373 dst : WR(write);
6374 mem : ISS(read);
6375 INS01 : ISS;
6376 LDST : WR;
6377 %}
6378
6379 // Load - reg, reg
6380 // Eg. LDR x0, [sp, x1]
6381 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6382 %{
6383 single_instruction;
6384 dst : WR(write);
6385 src : ISS(read);
6386 INS01 : ISS;
6387 LDST : WR;
6388 %}
6389
6390 //------- Store pipeline operations -----------------------
6391
6392 // Store - zr, mem
6393 // Eg. STR zr, <mem>
6394 pipe_class istore_mem(memory mem)
6395 %{
6396 single_instruction;
6397 mem : ISS(read);
6398 INS01 : ISS;
6399 LDST : WR;
6400 %}
6401
6402 // Store - reg, mem
6403 // Eg. STR x0, <mem>
6404 pipe_class istore_reg_mem(iRegI src, memory mem)
6405 %{
6406 single_instruction;
6407 mem : ISS(read);
6408 src : EX2(read);
6409 INS01 : ISS;
6410 LDST : WR;
6411 %}
6412
6413 // Store - reg, reg
6414 // Eg. STR x0, [sp, x1]
6415 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6416 %{
6417 single_instruction;
6418 dst : ISS(read);
6419 src : EX2(read);
6420 INS01 : ISS;
6421 LDST : WR;
6422 %}
6423
6424 //------- Store pipeline operations -----------------------
6425
6426 // Branch
6427 pipe_class pipe_branch()
6428 %{
6429 single_instruction;
6430 INS01 : ISS;
6431 BRANCH : EX1;
6432 %}
6433
6434 // Conditional branch
6435 pipe_class pipe_branch_cond(rFlagsReg cr)
6436 %{
6437 single_instruction;
6438 cr : EX1(read);
6439 INS01 : ISS;
6440 BRANCH : EX1;
6441 %}
6442
6443 // Compare & Branch
6444 // EG. CBZ/CBNZ
6445 pipe_class pipe_cmp_branch(iRegI op1)
6446 %{
6447 single_instruction;
6448 op1 : EX1(read);
6449 INS01 : ISS;
6450 BRANCH : EX1;
6451 %}
6452
6453 //------- Synchronisation operations ----------------------
6454
6455 // Any operation requiring serialization.
6456 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6457 pipe_class pipe_serial()
6458 %{
6459 single_instruction;
6460 force_serialization;
6461 fixed_latency(16);
6462 INS01 : ISS(2); // Cannot dual issue with any other instruction
6463 LDST : WR;
6464 %}
6465
6466 // Generic big/slow expanded idiom - also serialized
6467 pipe_class pipe_slow()
6468 %{
6469 instruction_count(10);
6470 multiple_bundles;
6471 force_serialization;
6472 fixed_latency(16);
6473 INS01 : ISS(2); // Cannot dual issue with any other instruction
6474 LDST : WR;
6475 %}
6476
6477 // Empty pipeline class
6478 pipe_class pipe_class_empty()
6479 %{
6480 single_instruction;
6481 fixed_latency(0);
6482 %}
6483
6484 // Default pipeline class.
6485 pipe_class pipe_class_default()
6486 %{
6487 single_instruction;
6488 fixed_latency(2);
6489 %}
6490
6491 // Pipeline class for compares.
6492 pipe_class pipe_class_compare()
6493 %{
6494 single_instruction;
6495 fixed_latency(16);
6496 %}
6497
6498 // Pipeline class for memory operations.
6499 pipe_class pipe_class_memory()
6500 %{
6501 single_instruction;
6502 fixed_latency(16);
6503 %}
6504
6505 // Pipeline class for call.
6506 pipe_class pipe_class_call()
6507 %{
6508 single_instruction;
6509 fixed_latency(100);
6510 %}
6511
6512 // Define the class for the Nop node.
6513 define %{
6514 MachNop = pipe_class_empty;
6515 %}
6516
6517 %}
6518 //----------INSTRUCTIONS-------------------------------------------------------
6519 //
6520 // match -- States which machine-independent subtree may be replaced
6521 // by this instruction.
6522 // ins_cost -- The estimated cost of this instruction is used by instruction
6523 // selection to identify a minimum cost tree of machine
6524 // instructions that matches a tree of machine-independent
6525 // instructions.
6526 // format -- A string providing the disassembly for this instruction.
6527 // The value of an instruction's operand may be inserted
6528 // by referring to it with a '$' prefix.
6529 // opcode -- Three instruction opcodes may be provided. These are referred
6530 // to within an encode class as $primary, $secondary, and $tertiary
6531 // rrspectively. The primary opcode is commonly used to
6532 // indicate the type of machine instruction, while secondary
6533 // and tertiary are often used for prefix options or addressing
6534 // modes.
6535 // ins_encode -- A list of encode classes with parameters. The encode class
6536 // name must have been defined in an 'enc_class' specification
6537 // in the encode section of the architecture description.
6538
6539 // ============================================================================
6540 // Memory (Load/Store) Instructions
6541
6542 // Load Instructions
6543
6544 // Load Byte (8 bit signed)
6545 instruct loadB(iRegINoSp dst, memory1 mem)
6546 %{
6547 match(Set dst (LoadB mem));
6548 predicate(!needs_acquiring_load(n));
6549
6550 ins_cost(4 * INSN_COST);
6551 format %{ "ldrsbw $dst, $mem\t# byte" %}
6552
6553 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6554
6555 ins_pipe(iload_reg_mem);
6556 %}
6557
6558 // Load Byte (8 bit signed) into long
6559 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6560 %{
6561 match(Set dst (ConvI2L (LoadB mem)));
6562 predicate(!needs_acquiring_load(n->in(1)));
6563
6564 ins_cost(4 * INSN_COST);
6565 format %{ "ldrsb $dst, $mem\t# byte" %}
6566
6567 ins_encode(aarch64_enc_ldrsb(dst, mem));
6568
6569 ins_pipe(iload_reg_mem);
6570 %}
6571
6572 // Load Byte (8 bit unsigned)
6573 instruct loadUB(iRegINoSp dst, memory1 mem)
6574 %{
6575 match(Set dst (LoadUB mem));
6576 predicate(!needs_acquiring_load(n));
6577
6578 ins_cost(4 * INSN_COST);
6579 format %{ "ldrbw $dst, $mem\t# byte" %}
6580
6581 ins_encode(aarch64_enc_ldrb(dst, mem));
6582
6583 ins_pipe(iload_reg_mem);
6584 %}
6585
6586 // Load Byte (8 bit unsigned) into long
6587 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6588 %{
6589 match(Set dst (ConvI2L (LoadUB mem)));
6590 predicate(!needs_acquiring_load(n->in(1)));
6591
6592 ins_cost(4 * INSN_COST);
6593 format %{ "ldrb $dst, $mem\t# byte" %}
6594
6595 ins_encode(aarch64_enc_ldrb(dst, mem));
6596
6597 ins_pipe(iload_reg_mem);
6598 %}
6599
6600 // Load Short (16 bit signed)
6601 instruct loadS(iRegINoSp dst, memory2 mem)
6602 %{
6603 match(Set dst (LoadS mem));
6604 predicate(!needs_acquiring_load(n));
6605
6606 ins_cost(4 * INSN_COST);
6607 format %{ "ldrshw $dst, $mem\t# short" %}
6608
6609 ins_encode(aarch64_enc_ldrshw(dst, mem));
6610
6611 ins_pipe(iload_reg_mem);
6612 %}
6613
6614 // Load Short (16 bit signed) into long
6615 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6616 %{
6617 match(Set dst (ConvI2L (LoadS mem)));
6618 predicate(!needs_acquiring_load(n->in(1)));
6619
6620 ins_cost(4 * INSN_COST);
6621 format %{ "ldrsh $dst, $mem\t# short" %}
6622
6623 ins_encode(aarch64_enc_ldrsh(dst, mem));
6624
6625 ins_pipe(iload_reg_mem);
6626 %}
6627
6628 // Load Char (16 bit unsigned)
6629 instruct loadUS(iRegINoSp dst, memory2 mem)
6630 %{
6631 match(Set dst (LoadUS mem));
6632 predicate(!needs_acquiring_load(n));
6633
6634 ins_cost(4 * INSN_COST);
6635 format %{ "ldrh $dst, $mem\t# short" %}
6636
6637 ins_encode(aarch64_enc_ldrh(dst, mem));
6638
6639 ins_pipe(iload_reg_mem);
6640 %}
6641
6642 // Load Short/Char (16 bit unsigned) into long
6643 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6644 %{
6645 match(Set dst (ConvI2L (LoadUS mem)));
6646 predicate(!needs_acquiring_load(n->in(1)));
6647
6648 ins_cost(4 * INSN_COST);
6649 format %{ "ldrh $dst, $mem\t# short" %}
6650
6651 ins_encode(aarch64_enc_ldrh(dst, mem));
6652
6653 ins_pipe(iload_reg_mem);
6654 %}
6655
6656 // Load Integer (32 bit signed)
6657 instruct loadI(iRegINoSp dst, memory4 mem)
6658 %{
6659 match(Set dst (LoadI mem));
6660 predicate(!needs_acquiring_load(n));
6661
6662 ins_cost(4 * INSN_COST);
6663 format %{ "ldrw $dst, $mem\t# int" %}
6664
6665 ins_encode(aarch64_enc_ldrw(dst, mem));
6666
6667 ins_pipe(iload_reg_mem);
6668 %}
6669
6670 // Load Integer (32 bit signed) into long
6671 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6672 %{
6673 match(Set dst (ConvI2L (LoadI mem)));
6674 predicate(!needs_acquiring_load(n->in(1)));
6675
6676 ins_cost(4 * INSN_COST);
6677 format %{ "ldrsw $dst, $mem\t# int" %}
6678
6679 ins_encode(aarch64_enc_ldrsw(dst, mem));
6680
6681 ins_pipe(iload_reg_mem);
6682 %}
6683
6684 // Load Integer (32 bit unsigned) into long
6685 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6686 %{
6687 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6688 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6689
6690 ins_cost(4 * INSN_COST);
6691 format %{ "ldrw $dst, $mem\t# int" %}
6692
6693 ins_encode(aarch64_enc_ldrw(dst, mem));
6694
6695 ins_pipe(iload_reg_mem);
6696 %}
6697
6698 // Load Long (64 bit signed)
6699 instruct loadL(iRegLNoSp dst, memory8 mem)
6700 %{
6701 match(Set dst (LoadL mem));
6702 predicate(!needs_acquiring_load(n));
6703
6704 ins_cost(4 * INSN_COST);
6705 format %{ "ldr $dst, $mem\t# int" %}
6706
6707 ins_encode(aarch64_enc_ldr(dst, mem));
6708
6709 ins_pipe(iload_reg_mem);
6710 %}
6711
6712 // Load Range
6713 instruct loadRange(iRegINoSp dst, memory4 mem)
6714 %{
6715 match(Set dst (LoadRange mem));
6716
6717 ins_cost(4 * INSN_COST);
6718 format %{ "ldrw $dst, $mem\t# range" %}
6719
6720 ins_encode(aarch64_enc_ldrw(dst, mem));
6721
6722 ins_pipe(iload_reg_mem);
6723 %}
6724
6725 // Load Pointer
6726 instruct loadP(iRegPNoSp dst, memory8 mem)
6727 %{
6728 match(Set dst (LoadP mem));
6729 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6730
6731 ins_cost(4 * INSN_COST);
6732 format %{ "ldr $dst, $mem\t# ptr" %}
6733
6734 ins_encode(aarch64_enc_ldr(dst, mem));
6735
6736 ins_pipe(iload_reg_mem);
6737 %}
6738
6739 // Load Compressed Pointer
6740 instruct loadN(iRegNNoSp dst, memory4 mem)
6741 %{
6742 match(Set dst (LoadN mem));
6743 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6744
6745 ins_cost(4 * INSN_COST);
6746 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6747
6748 ins_encode(aarch64_enc_ldrw(dst, mem));
6749
6750 ins_pipe(iload_reg_mem);
6751 %}
6752
6753 // Load Klass Pointer
6754 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6755 %{
6756 match(Set dst (LoadKlass mem));
6757 predicate(!needs_acquiring_load(n));
6758
6759 ins_cost(4 * INSN_COST);
6760 format %{ "ldr $dst, $mem\t# class" %}
6761
6762 ins_encode(aarch64_enc_ldr(dst, mem));
6763
6764 ins_pipe(iload_reg_mem);
6765 %}
6766
6767 // Load Narrow Klass Pointer
6768 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6769 %{
6770 match(Set dst (LoadNKlass mem));
6771 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6772
6773 ins_cost(4 * INSN_COST);
6774 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6775
6776 ins_encode(aarch64_enc_ldrw(dst, mem));
6777
6778 ins_pipe(iload_reg_mem);
6779 %}
6780
6781 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6782 %{
6783 match(Set dst (LoadNKlass mem));
6784 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6785
6786 ins_cost(4 * INSN_COST);
6787 format %{
6788 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6789 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6790 %}
6791 ins_encode %{
6792 // inlined aarch64_enc_ldrw
6793 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6794 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6795 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6796 %}
6797 ins_pipe(iload_reg_mem);
6798 %}
6799
6800 // Load Float
6801 instruct loadF(vRegF dst, memory4 mem)
6802 %{
6803 match(Set dst (LoadF mem));
6804 predicate(!needs_acquiring_load(n));
6805
6806 ins_cost(4 * INSN_COST);
6807 format %{ "ldrs $dst, $mem\t# float" %}
6808
6809 ins_encode( aarch64_enc_ldrs(dst, mem) );
6810
6811 ins_pipe(pipe_class_memory);
6812 %}
6813
6814 // Load Double
6815 instruct loadD(vRegD dst, memory8 mem)
6816 %{
6817 match(Set dst (LoadD mem));
6818 predicate(!needs_acquiring_load(n));
6819
6820 ins_cost(4 * INSN_COST);
6821 format %{ "ldrd $dst, $mem\t# double" %}
6822
6823 ins_encode( aarch64_enc_ldrd(dst, mem) );
6824
6825 ins_pipe(pipe_class_memory);
6826 %}
6827
6828
6829 // Load Int Constant
6830 instruct loadConI(iRegINoSp dst, immI src)
6831 %{
6832 match(Set dst src);
6833
6834 ins_cost(INSN_COST);
6835 format %{ "mov $dst, $src\t# int" %}
6836
6837 ins_encode( aarch64_enc_movw_imm(dst, src) );
6838
6839 ins_pipe(ialu_imm);
6840 %}
6841
6842 // Load Long Constant
6843 instruct loadConL(iRegLNoSp dst, immL src)
6844 %{
6845 match(Set dst src);
6846
6847 ins_cost(INSN_COST);
6848 format %{ "mov $dst, $src\t# long" %}
6849
6850 ins_encode( aarch64_enc_mov_imm(dst, src) );
6851
6852 ins_pipe(ialu_imm);
6853 %}
6854
6855 // Load Pointer Constant
6856
6857 instruct loadConP(iRegPNoSp dst, immP con)
6858 %{
6859 match(Set dst con);
6860
6861 ins_cost(INSN_COST * 4);
6862 format %{
6863 "mov $dst, $con\t# ptr\n\t"
6864 %}
6865
6866 ins_encode(aarch64_enc_mov_p(dst, con));
6867
6868 ins_pipe(ialu_imm);
6869 %}
6870
6871 // Load Null Pointer Constant
6872
6873 instruct loadConP0(iRegPNoSp dst, immP0 con)
6874 %{
6875 match(Set dst con);
6876
6877 ins_cost(INSN_COST);
6878 format %{ "mov $dst, $con\t# nullptr ptr" %}
6879
6880 ins_encode(aarch64_enc_mov_p0(dst, con));
6881
6882 ins_pipe(ialu_imm);
6883 %}
6884
6885 // Load Pointer Constant One
6886
6887 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6888 %{
6889 match(Set dst con);
6890
6891 ins_cost(INSN_COST);
6892 format %{ "mov $dst, $con\t# nullptr ptr" %}
6893
6894 ins_encode(aarch64_enc_mov_p1(dst, con));
6895
6896 ins_pipe(ialu_imm);
6897 %}
6898
6899 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6900 %{
6901 match(Set dst con);
6902
6903 ins_cost(INSN_COST);
6904 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6905
6906 ins_encode %{
6907 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6908 %}
6909
6910 ins_pipe(ialu_imm);
6911 %}
6912
6913 // Load Narrow Pointer Constant
6914
6915 instruct loadConN(iRegNNoSp dst, immN con)
6916 %{
6917 match(Set dst con);
6918
6919 ins_cost(INSN_COST * 4);
6920 format %{ "mov $dst, $con\t# compressed ptr" %}
6921
6922 ins_encode(aarch64_enc_mov_n(dst, con));
6923
6924 ins_pipe(ialu_imm);
6925 %}
6926
6927 // Load Narrow Null Pointer Constant
6928
6929 instruct loadConN0(iRegNNoSp dst, immN0 con)
6930 %{
6931 match(Set dst con);
6932
6933 ins_cost(INSN_COST);
6934 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6935
6936 ins_encode(aarch64_enc_mov_n0(dst, con));
6937
6938 ins_pipe(ialu_imm);
6939 %}
6940
6941 // Load Narrow Klass Constant
6942
6943 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6944 %{
6945 match(Set dst con);
6946
6947 ins_cost(INSN_COST);
6948 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6949
6950 ins_encode(aarch64_enc_mov_nk(dst, con));
6951
6952 ins_pipe(ialu_imm);
6953 %}
6954
6955 // Load Packed Float Constant
6956
6957 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6958 match(Set dst con);
6959 ins_cost(INSN_COST * 4);
6960 format %{ "fmovs $dst, $con"%}
6961 ins_encode %{
6962 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6963 %}
6964
6965 ins_pipe(fp_imm_s);
6966 %}
6967
6968 // Load Float Constant
6969
6970 instruct loadConF(vRegF dst, immF con) %{
6971 match(Set dst con);
6972
6973 ins_cost(INSN_COST * 4);
6974
6975 format %{
6976 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6977 %}
6978
6979 ins_encode %{
6980 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6981 %}
6982
6983 ins_pipe(fp_load_constant_s);
6984 %}
6985
6986 // Load Packed Double Constant
6987
6988 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6989 match(Set dst con);
6990 ins_cost(INSN_COST);
6991 format %{ "fmovd $dst, $con"%}
6992 ins_encode %{
6993 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6994 %}
6995
6996 ins_pipe(fp_imm_d);
6997 %}
6998
6999 // Load Double Constant
7000
7001 instruct loadConD(vRegD dst, immD con) %{
7002 match(Set dst con);
7003
7004 ins_cost(INSN_COST * 5);
7005 format %{
7006 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7007 %}
7008
7009 ins_encode %{
7010 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7011 %}
7012
7013 ins_pipe(fp_load_constant_d);
7014 %}
7015
7016 // Load Half Float Constant
7017 instruct loadConH(vRegF dst, immH con) %{
7018 match(Set dst con);
7019 format %{ "mov rscratch1, $con\n\t"
7020 "fmov $dst, rscratch1"
7021 %}
7022 ins_encode %{
7023 __ movw(rscratch1, (uint32_t)$con$$constant);
7024 __ fmovs($dst$$FloatRegister, rscratch1);
7025 %}
7026 ins_pipe(pipe_class_default);
7027 %}
7028
7029 // Store Instructions
7030
7031 // Store Byte
7032 instruct storeB(iRegIorL2I src, memory1 mem)
7033 %{
7034 match(Set mem (StoreB mem src));
7035 predicate(!needs_releasing_store(n));
7036
7037 ins_cost(INSN_COST);
7038 format %{ "strb $src, $mem\t# byte" %}
7039
7040 ins_encode(aarch64_enc_strb(src, mem));
7041
7042 ins_pipe(istore_reg_mem);
7043 %}
7044
7045
7046 instruct storeimmB0(immI0 zero, memory1 mem)
7047 %{
7048 match(Set mem (StoreB mem zero));
7049 predicate(!needs_releasing_store(n));
7050
7051 ins_cost(INSN_COST);
7052 format %{ "strb rscractch2, $mem\t# byte" %}
7053
7054 ins_encode(aarch64_enc_strb0(mem));
7055
7056 ins_pipe(istore_mem);
7057 %}
7058
7059 // Store Char/Short
7060 instruct storeC(iRegIorL2I src, memory2 mem)
7061 %{
7062 match(Set mem (StoreC mem src));
7063 predicate(!needs_releasing_store(n));
7064
7065 ins_cost(INSN_COST);
7066 format %{ "strh $src, $mem\t# short" %}
7067
7068 ins_encode(aarch64_enc_strh(src, mem));
7069
7070 ins_pipe(istore_reg_mem);
7071 %}
7072
7073 instruct storeimmC0(immI0 zero, memory2 mem)
7074 %{
7075 match(Set mem (StoreC mem zero));
7076 predicate(!needs_releasing_store(n));
7077
7078 ins_cost(INSN_COST);
7079 format %{ "strh zr, $mem\t# short" %}
7080
7081 ins_encode(aarch64_enc_strh0(mem));
7082
7083 ins_pipe(istore_mem);
7084 %}
7085
7086 // Store Integer
7087
7088 instruct storeI(iRegIorL2I src, memory4 mem)
7089 %{
7090 match(Set mem(StoreI mem src));
7091 predicate(!needs_releasing_store(n));
7092
7093 ins_cost(INSN_COST);
7094 format %{ "strw $src, $mem\t# int" %}
7095
7096 ins_encode(aarch64_enc_strw(src, mem));
7097
7098 ins_pipe(istore_reg_mem);
7099 %}
7100
7101 instruct storeimmI0(immI0 zero, memory4 mem)
7102 %{
7103 match(Set mem(StoreI mem zero));
7104 predicate(!needs_releasing_store(n));
7105
7106 ins_cost(INSN_COST);
7107 format %{ "strw zr, $mem\t# int" %}
7108
7109 ins_encode(aarch64_enc_strw0(mem));
7110
7111 ins_pipe(istore_mem);
7112 %}
7113
7114 // Store Long (64 bit signed)
7115 instruct storeL(iRegL src, memory8 mem)
7116 %{
7117 match(Set mem (StoreL mem src));
7118 predicate(!needs_releasing_store(n));
7119
7120 ins_cost(INSN_COST);
7121 format %{ "str $src, $mem\t# int" %}
7122
7123 ins_encode(aarch64_enc_str(src, mem));
7124
7125 ins_pipe(istore_reg_mem);
7126 %}
7127
7128 // Store Long (64 bit signed)
7129 instruct storeimmL0(immL0 zero, memory8 mem)
7130 %{
7131 match(Set mem (StoreL mem zero));
7132 predicate(!needs_releasing_store(n));
7133
7134 ins_cost(INSN_COST);
7135 format %{ "str zr, $mem\t# int" %}
7136
7137 ins_encode(aarch64_enc_str0(mem));
7138
7139 ins_pipe(istore_mem);
7140 %}
7141
7142 // Store Pointer
7143 instruct storeP(iRegP src, memory8 mem)
7144 %{
7145 match(Set mem (StoreP mem src));
7146 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7147
7148 ins_cost(INSN_COST);
7149 format %{ "str $src, $mem\t# ptr" %}
7150
7151 ins_encode(aarch64_enc_str(src, mem));
7152
7153 ins_pipe(istore_reg_mem);
7154 %}
7155
7156 // Store Pointer
7157 instruct storeimmP0(immP0 zero, memory8 mem)
7158 %{
7159 match(Set mem (StoreP mem zero));
7160 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7161
7162 ins_cost(INSN_COST);
7163 format %{ "str zr, $mem\t# ptr" %}
7164
7165 ins_encode(aarch64_enc_str0(mem));
7166
7167 ins_pipe(istore_mem);
7168 %}
7169
7170 // Store Compressed Pointer
7171 instruct storeN(iRegN src, memory4 mem)
7172 %{
7173 match(Set mem (StoreN mem src));
7174 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7175
7176 ins_cost(INSN_COST);
7177 format %{ "strw $src, $mem\t# compressed ptr" %}
7178
7179 ins_encode(aarch64_enc_strw(src, mem));
7180
7181 ins_pipe(istore_reg_mem);
7182 %}
7183
7184 instruct storeImmN0(immN0 zero, memory4 mem)
7185 %{
7186 match(Set mem (StoreN mem zero));
7187 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7188
7189 ins_cost(INSN_COST);
7190 format %{ "strw zr, $mem\t# compressed ptr" %}
7191
7192 ins_encode(aarch64_enc_strw0(mem));
7193
7194 ins_pipe(istore_mem);
7195 %}
7196
7197 // Store Float
7198 instruct storeF(vRegF src, memory4 mem)
7199 %{
7200 match(Set mem (StoreF mem src));
7201 predicate(!needs_releasing_store(n));
7202
7203 ins_cost(INSN_COST);
7204 format %{ "strs $src, $mem\t# float" %}
7205
7206 ins_encode( aarch64_enc_strs(src, mem) );
7207
7208 ins_pipe(pipe_class_memory);
7209 %}
7210
7211 // TODO
7212 // implement storeImmF0 and storeFImmPacked
7213
7214 // Store Double
7215 instruct storeD(vRegD src, memory8 mem)
7216 %{
7217 match(Set mem (StoreD mem src));
7218 predicate(!needs_releasing_store(n));
7219
7220 ins_cost(INSN_COST);
7221 format %{ "strd $src, $mem\t# double" %}
7222
7223 ins_encode( aarch64_enc_strd(src, mem) );
7224
7225 ins_pipe(pipe_class_memory);
7226 %}
7227
7228 // Store Compressed Klass Pointer
7229 instruct storeNKlass(iRegN src, memory4 mem)
7230 %{
7231 predicate(!needs_releasing_store(n));
7232 match(Set mem (StoreNKlass mem src));
7233
7234 ins_cost(INSN_COST);
7235 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7236
7237 ins_encode(aarch64_enc_strw(src, mem));
7238
7239 ins_pipe(istore_reg_mem);
7240 %}
7241
7242 // TODO
7243 // implement storeImmD0 and storeDImmPacked
7244
7245 // prefetch instructions
7246 // Must be safe to execute with invalid address (cannot fault).
7247
7248 instruct prefetchalloc( memory8 mem ) %{
7249 match(PrefetchAllocation mem);
7250
7251 ins_cost(INSN_COST);
7252 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7253
7254 ins_encode( aarch64_enc_prefetchw(mem) );
7255
7256 ins_pipe(iload_prefetch);
7257 %}
7258
7259 // ---------------- volatile loads and stores ----------------
7260
7261 // Load Byte (8 bit signed)
7262 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7263 %{
7264 match(Set dst (LoadB mem));
7265
7266 ins_cost(VOLATILE_REF_COST);
7267 format %{ "ldarsb $dst, $mem\t# byte" %}
7268
7269 ins_encode(aarch64_enc_ldarsb(dst, mem));
7270
7271 ins_pipe(pipe_serial);
7272 %}
7273
7274 // Load Byte (8 bit signed) into long
7275 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7276 %{
7277 match(Set dst (ConvI2L (LoadB mem)));
7278
7279 ins_cost(VOLATILE_REF_COST);
7280 format %{ "ldarsb $dst, $mem\t# byte" %}
7281
7282 ins_encode(aarch64_enc_ldarsb(dst, mem));
7283
7284 ins_pipe(pipe_serial);
7285 %}
7286
7287 // Load Byte (8 bit unsigned)
7288 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7289 %{
7290 match(Set dst (LoadUB mem));
7291
7292 ins_cost(VOLATILE_REF_COST);
7293 format %{ "ldarb $dst, $mem\t# byte" %}
7294
7295 ins_encode(aarch64_enc_ldarb(dst, mem));
7296
7297 ins_pipe(pipe_serial);
7298 %}
7299
7300 // Load Byte (8 bit unsigned) into long
7301 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7302 %{
7303 match(Set dst (ConvI2L (LoadUB mem)));
7304
7305 ins_cost(VOLATILE_REF_COST);
7306 format %{ "ldarb $dst, $mem\t# byte" %}
7307
7308 ins_encode(aarch64_enc_ldarb(dst, mem));
7309
7310 ins_pipe(pipe_serial);
7311 %}
7312
7313 // Load Short (16 bit signed)
7314 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7315 %{
7316 match(Set dst (LoadS mem));
7317
7318 ins_cost(VOLATILE_REF_COST);
7319 format %{ "ldarshw $dst, $mem\t# short" %}
7320
7321 ins_encode(aarch64_enc_ldarshw(dst, mem));
7322
7323 ins_pipe(pipe_serial);
7324 %}
7325
7326 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7327 %{
7328 match(Set dst (LoadUS mem));
7329
7330 ins_cost(VOLATILE_REF_COST);
7331 format %{ "ldarhw $dst, $mem\t# short" %}
7332
7333 ins_encode(aarch64_enc_ldarhw(dst, mem));
7334
7335 ins_pipe(pipe_serial);
7336 %}
7337
7338 // Load Short/Char (16 bit unsigned) into long
7339 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7340 %{
7341 match(Set dst (ConvI2L (LoadUS mem)));
7342
7343 ins_cost(VOLATILE_REF_COST);
7344 format %{ "ldarh $dst, $mem\t# short" %}
7345
7346 ins_encode(aarch64_enc_ldarh(dst, mem));
7347
7348 ins_pipe(pipe_serial);
7349 %}
7350
7351 // Load Short/Char (16 bit signed) into long
7352 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7353 %{
7354 match(Set dst (ConvI2L (LoadS mem)));
7355
7356 ins_cost(VOLATILE_REF_COST);
7357 format %{ "ldarh $dst, $mem\t# short" %}
7358
7359 ins_encode(aarch64_enc_ldarsh(dst, mem));
7360
7361 ins_pipe(pipe_serial);
7362 %}
7363
7364 // Load Integer (32 bit signed)
7365 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7366 %{
7367 match(Set dst (LoadI mem));
7368
7369 ins_cost(VOLATILE_REF_COST);
7370 format %{ "ldarw $dst, $mem\t# int" %}
7371
7372 ins_encode(aarch64_enc_ldarw(dst, mem));
7373
7374 ins_pipe(pipe_serial);
7375 %}
7376
7377 // Load Integer (32 bit unsigned) into long
7378 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7379 %{
7380 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7381
7382 ins_cost(VOLATILE_REF_COST);
7383 format %{ "ldarw $dst, $mem\t# int" %}
7384
7385 ins_encode(aarch64_enc_ldarw(dst, mem));
7386
7387 ins_pipe(pipe_serial);
7388 %}
7389
7390 // Load Long (64 bit signed)
7391 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7392 %{
7393 match(Set dst (LoadL mem));
7394
7395 ins_cost(VOLATILE_REF_COST);
7396 format %{ "ldar $dst, $mem\t# int" %}
7397
7398 ins_encode(aarch64_enc_ldar(dst, mem));
7399
7400 ins_pipe(pipe_serial);
7401 %}
7402
7403 // Load Pointer
7404 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7405 %{
7406 match(Set dst (LoadP mem));
7407 predicate(n->as_Load()->barrier_data() == 0);
7408
7409 ins_cost(VOLATILE_REF_COST);
7410 format %{ "ldar $dst, $mem\t# ptr" %}
7411
7412 ins_encode(aarch64_enc_ldar(dst, mem));
7413
7414 ins_pipe(pipe_serial);
7415 %}
7416
7417 // Load Compressed Pointer
7418 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7419 %{
7420 match(Set dst (LoadN mem));
7421 predicate(n->as_Load()->barrier_data() == 0);
7422
7423 ins_cost(VOLATILE_REF_COST);
7424 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7425
7426 ins_encode(aarch64_enc_ldarw(dst, mem));
7427
7428 ins_pipe(pipe_serial);
7429 %}
7430
7431 // Load Float
7432 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7433 %{
7434 match(Set dst (LoadF mem));
7435
7436 ins_cost(VOLATILE_REF_COST);
7437 format %{ "ldars $dst, $mem\t# float" %}
7438
7439 ins_encode( aarch64_enc_fldars(dst, mem) );
7440
7441 ins_pipe(pipe_serial);
7442 %}
7443
7444 // Load Double
7445 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7446 %{
7447 match(Set dst (LoadD mem));
7448
7449 ins_cost(VOLATILE_REF_COST);
7450 format %{ "ldard $dst, $mem\t# double" %}
7451
7452 ins_encode( aarch64_enc_fldard(dst, mem) );
7453
7454 ins_pipe(pipe_serial);
7455 %}
7456
7457 // Store Byte
7458 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7459 %{
7460 match(Set mem (StoreB mem src));
7461
7462 ins_cost(VOLATILE_REF_COST);
7463 format %{ "stlrb $src, $mem\t# byte" %}
7464
7465 ins_encode(aarch64_enc_stlrb(src, mem));
7466
7467 ins_pipe(pipe_class_memory);
7468 %}
7469
7470 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7471 %{
7472 match(Set mem (StoreB mem zero));
7473
7474 ins_cost(VOLATILE_REF_COST);
7475 format %{ "stlrb zr, $mem\t# byte" %}
7476
7477 ins_encode(aarch64_enc_stlrb0(mem));
7478
7479 ins_pipe(pipe_class_memory);
7480 %}
7481
7482 // Store Char/Short
7483 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7484 %{
7485 match(Set mem (StoreC mem src));
7486
7487 ins_cost(VOLATILE_REF_COST);
7488 format %{ "stlrh $src, $mem\t# short" %}
7489
7490 ins_encode(aarch64_enc_stlrh(src, mem));
7491
7492 ins_pipe(pipe_class_memory);
7493 %}
7494
7495 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7496 %{
7497 match(Set mem (StoreC mem zero));
7498
7499 ins_cost(VOLATILE_REF_COST);
7500 format %{ "stlrh zr, $mem\t# short" %}
7501
7502 ins_encode(aarch64_enc_stlrh0(mem));
7503
7504 ins_pipe(pipe_class_memory);
7505 %}
7506
7507 // Store Integer
7508
7509 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7510 %{
7511 match(Set mem(StoreI mem src));
7512
7513 ins_cost(VOLATILE_REF_COST);
7514 format %{ "stlrw $src, $mem\t# int" %}
7515
7516 ins_encode(aarch64_enc_stlrw(src, mem));
7517
7518 ins_pipe(pipe_class_memory);
7519 %}
7520
7521 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7522 %{
7523 match(Set mem(StoreI mem zero));
7524
7525 ins_cost(VOLATILE_REF_COST);
7526 format %{ "stlrw zr, $mem\t# int" %}
7527
7528 ins_encode(aarch64_enc_stlrw0(mem));
7529
7530 ins_pipe(pipe_class_memory);
7531 %}
7532
7533 // Store Long (64 bit signed)
7534 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7535 %{
7536 match(Set mem (StoreL mem src));
7537
7538 ins_cost(VOLATILE_REF_COST);
7539 format %{ "stlr $src, $mem\t# int" %}
7540
7541 ins_encode(aarch64_enc_stlr(src, mem));
7542
7543 ins_pipe(pipe_class_memory);
7544 %}
7545
7546 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7547 %{
7548 match(Set mem (StoreL mem zero));
7549
7550 ins_cost(VOLATILE_REF_COST);
7551 format %{ "stlr zr, $mem\t# int" %}
7552
7553 ins_encode(aarch64_enc_stlr0(mem));
7554
7555 ins_pipe(pipe_class_memory);
7556 %}
7557
7558 // Store Pointer
7559 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7560 %{
7561 match(Set mem (StoreP mem src));
7562 predicate(n->as_Store()->barrier_data() == 0);
7563
7564 ins_cost(VOLATILE_REF_COST);
7565 format %{ "stlr $src, $mem\t# ptr" %}
7566
7567 ins_encode(aarch64_enc_stlr(src, mem));
7568
7569 ins_pipe(pipe_class_memory);
7570 %}
7571
7572 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7573 %{
7574 match(Set mem (StoreP mem zero));
7575 predicate(n->as_Store()->barrier_data() == 0);
7576
7577 ins_cost(VOLATILE_REF_COST);
7578 format %{ "stlr zr, $mem\t# ptr" %}
7579
7580 ins_encode(aarch64_enc_stlr0(mem));
7581
7582 ins_pipe(pipe_class_memory);
7583 %}
7584
7585 // Store Compressed Pointer
7586 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7587 %{
7588 match(Set mem (StoreN mem src));
7589 predicate(n->as_Store()->barrier_data() == 0);
7590
7591 ins_cost(VOLATILE_REF_COST);
7592 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7593
7594 ins_encode(aarch64_enc_stlrw(src, mem));
7595
7596 ins_pipe(pipe_class_memory);
7597 %}
7598
7599 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7600 %{
7601 match(Set mem (StoreN mem zero));
7602 predicate(n->as_Store()->barrier_data() == 0);
7603
7604 ins_cost(VOLATILE_REF_COST);
7605 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7606
7607 ins_encode(aarch64_enc_stlrw0(mem));
7608
7609 ins_pipe(pipe_class_memory);
7610 %}
7611
7612 // Store Float
7613 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7614 %{
7615 match(Set mem (StoreF mem src));
7616
7617 ins_cost(VOLATILE_REF_COST);
7618 format %{ "stlrs $src, $mem\t# float" %}
7619
7620 ins_encode( aarch64_enc_fstlrs(src, mem) );
7621
7622 ins_pipe(pipe_class_memory);
7623 %}
7624
7625 // TODO
7626 // implement storeImmF0 and storeFImmPacked
7627
7628 // Store Double
7629 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7630 %{
7631 match(Set mem (StoreD mem src));
7632
7633 ins_cost(VOLATILE_REF_COST);
7634 format %{ "stlrd $src, $mem\t# double" %}
7635
7636 ins_encode( aarch64_enc_fstlrd(src, mem) );
7637
7638 ins_pipe(pipe_class_memory);
7639 %}
7640
7641 // ---------------- end of volatile loads and stores ----------------
7642
7643 instruct cacheWB(indirect addr)
7644 %{
7645 predicate(VM_Version::supports_data_cache_line_flush());
7646 match(CacheWB addr);
7647
7648 ins_cost(100);
7649 format %{"cache wb $addr" %}
7650 ins_encode %{
7651 assert($addr->index_position() < 0, "should be");
7652 assert($addr$$disp == 0, "should be");
7653 __ cache_wb(Address($addr$$base$$Register, 0));
7654 %}
7655 ins_pipe(pipe_slow); // XXX
7656 %}
7657
7658 instruct cacheWBPreSync()
7659 %{
7660 predicate(VM_Version::supports_data_cache_line_flush());
7661 match(CacheWBPreSync);
7662
7663 ins_cost(100);
7664 format %{"cache wb presync" %}
7665 ins_encode %{
7666 __ cache_wbsync(true);
7667 %}
7668 ins_pipe(pipe_slow); // XXX
7669 %}
7670
7671 instruct cacheWBPostSync()
7672 %{
7673 predicate(VM_Version::supports_data_cache_line_flush());
7674 match(CacheWBPostSync);
7675
7676 ins_cost(100);
7677 format %{"cache wb postsync" %}
7678 ins_encode %{
7679 __ cache_wbsync(false);
7680 %}
7681 ins_pipe(pipe_slow); // XXX
7682 %}
7683
7684 // ============================================================================
7685 // BSWAP Instructions
7686
7687 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7688 match(Set dst (ReverseBytesI src));
7689
7690 ins_cost(INSN_COST);
7691 format %{ "revw $dst, $src" %}
7692
7693 ins_encode %{
7694 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7695 %}
7696
7697 ins_pipe(ialu_reg);
7698 %}
7699
7700 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7701 match(Set dst (ReverseBytesL src));
7702
7703 ins_cost(INSN_COST);
7704 format %{ "rev $dst, $src" %}
7705
7706 ins_encode %{
7707 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7708 %}
7709
7710 ins_pipe(ialu_reg);
7711 %}
7712
7713 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7714 match(Set dst (ReverseBytesUS src));
7715
7716 ins_cost(INSN_COST);
7717 format %{ "rev16w $dst, $src" %}
7718
7719 ins_encode %{
7720 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7721 %}
7722
7723 ins_pipe(ialu_reg);
7724 %}
7725
7726 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7727 match(Set dst (ReverseBytesS src));
7728
7729 ins_cost(INSN_COST);
7730 format %{ "rev16w $dst, $src\n\t"
7731 "sbfmw $dst, $dst, #0, #15" %}
7732
7733 ins_encode %{
7734 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7735 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7736 %}
7737
7738 ins_pipe(ialu_reg);
7739 %}
7740
7741 // ============================================================================
7742 // Zero Count Instructions
7743
7744 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7745 match(Set dst (CountLeadingZerosI src));
7746
7747 ins_cost(INSN_COST);
7748 format %{ "clzw $dst, $src" %}
7749 ins_encode %{
7750 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7751 %}
7752
7753 ins_pipe(ialu_reg);
7754 %}
7755
7756 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7757 match(Set dst (CountLeadingZerosL src));
7758
7759 ins_cost(INSN_COST);
7760 format %{ "clz $dst, $src" %}
7761 ins_encode %{
7762 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7763 %}
7764
7765 ins_pipe(ialu_reg);
7766 %}
7767
7768 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7769 match(Set dst (CountTrailingZerosI src));
7770
7771 ins_cost(INSN_COST * 2);
7772 format %{ "rbitw $dst, $src\n\t"
7773 "clzw $dst, $dst" %}
7774 ins_encode %{
7775 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7776 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7777 %}
7778
7779 ins_pipe(ialu_reg);
7780 %}
7781
7782 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7783 match(Set dst (CountTrailingZerosL src));
7784
7785 ins_cost(INSN_COST * 2);
7786 format %{ "rbit $dst, $src\n\t"
7787 "clz $dst, $dst" %}
7788 ins_encode %{
7789 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7790 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7791 %}
7792
7793 ins_pipe(ialu_reg);
7794 %}
7795
7796 //---------- Population Count Instructions -------------------------------------
7797 //
7798
7799 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7800 match(Set dst (PopCountI src));
7801 effect(TEMP tmp);
7802 ins_cost(INSN_COST * 13);
7803
7804 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7805 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7806 "addv $tmp, $tmp\t# vector (8B)\n\t"
7807 "mov $dst, $tmp\t# vector (1D)" %}
7808 ins_encode %{
7809 __ fmovs($tmp$$FloatRegister, $src$$Register);
7810 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7811 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7812 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7813 %}
7814
7815 ins_pipe(pipe_class_default);
7816 %}
7817
7818 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7819 match(Set dst (PopCountI (LoadI mem)));
7820 effect(TEMP tmp);
7821 ins_cost(INSN_COST * 13);
7822
7823 format %{ "ldrs $tmp, $mem\n\t"
7824 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7825 "addv $tmp, $tmp\t# vector (8B)\n\t"
7826 "mov $dst, $tmp\t# vector (1D)" %}
7827 ins_encode %{
7828 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7829 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7830 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7831 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7832 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7833 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7834 %}
7835
7836 ins_pipe(pipe_class_default);
7837 %}
7838
7839 // Note: Long.bitCount(long) returns an int.
7840 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7841 match(Set dst (PopCountL src));
7842 effect(TEMP tmp);
7843 ins_cost(INSN_COST * 13);
7844
7845 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7846 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7847 "addv $tmp, $tmp\t# vector (8B)\n\t"
7848 "mov $dst, $tmp\t# vector (1D)" %}
7849 ins_encode %{
7850 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7851 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7852 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7853 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7854 %}
7855
7856 ins_pipe(pipe_class_default);
7857 %}
7858
7859 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7860 match(Set dst (PopCountL (LoadL mem)));
7861 effect(TEMP tmp);
7862 ins_cost(INSN_COST * 13);
7863
7864 format %{ "ldrd $tmp, $mem\n\t"
7865 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7866 "addv $tmp, $tmp\t# vector (8B)\n\t"
7867 "mov $dst, $tmp\t# vector (1D)" %}
7868 ins_encode %{
7869 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7870 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7871 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7872 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7873 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7874 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7875 %}
7876
7877 ins_pipe(pipe_class_default);
7878 %}
7879
7880 // ============================================================================
7881 // VerifyVectorAlignment Instruction
7882
7883 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7884 match(Set addr (VerifyVectorAlignment addr mask));
7885 effect(KILL cr);
7886 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7887 ins_encode %{
7888 Label Lskip;
7889 // check if masked bits of addr are zero
7890 __ tst($addr$$Register, $mask$$constant);
7891 __ br(Assembler::EQ, Lskip);
7892 __ stop("verify_vector_alignment found a misaligned vector memory access");
7893 __ bind(Lskip);
7894 %}
7895 ins_pipe(pipe_slow);
7896 %}
7897
7898 // ============================================================================
7899 // MemBar Instruction
7900
7901 instruct load_fence() %{
7902 match(LoadFence);
7903 ins_cost(VOLATILE_REF_COST);
7904
7905 format %{ "load_fence" %}
7906
7907 ins_encode %{
7908 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7909 %}
7910 ins_pipe(pipe_serial);
7911 %}
7912
7913 instruct unnecessary_membar_acquire() %{
7914 predicate(unnecessary_acquire(n));
7915 match(MemBarAcquire);
7916 ins_cost(0);
7917
7918 format %{ "membar_acquire (elided)" %}
7919
7920 ins_encode %{
7921 __ block_comment("membar_acquire (elided)");
7922 %}
7923
7924 ins_pipe(pipe_class_empty);
7925 %}
7926
7927 instruct membar_acquire() %{
7928 match(MemBarAcquire);
7929 ins_cost(VOLATILE_REF_COST);
7930
7931 format %{ "membar_acquire\n\t"
7932 "dmb ishld" %}
7933
7934 ins_encode %{
7935 __ block_comment("membar_acquire");
7936 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7937 %}
7938
7939 ins_pipe(pipe_serial);
7940 %}
7941
7942
7943 instruct membar_acquire_lock() %{
7944 match(MemBarAcquireLock);
7945 ins_cost(VOLATILE_REF_COST);
7946
7947 format %{ "membar_acquire_lock (elided)" %}
7948
7949 ins_encode %{
7950 __ block_comment("membar_acquire_lock (elided)");
7951 %}
7952
7953 ins_pipe(pipe_serial);
7954 %}
7955
7956 instruct store_fence() %{
7957 match(StoreFence);
7958 ins_cost(VOLATILE_REF_COST);
7959
7960 format %{ "store_fence" %}
7961
7962 ins_encode %{
7963 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7964 %}
7965 ins_pipe(pipe_serial);
7966 %}
7967
7968 instruct unnecessary_membar_release() %{
7969 predicate(unnecessary_release(n));
7970 match(MemBarRelease);
7971 ins_cost(0);
7972
7973 format %{ "membar_release (elided)" %}
7974
7975 ins_encode %{
7976 __ block_comment("membar_release (elided)");
7977 %}
7978 ins_pipe(pipe_serial);
7979 %}
7980
7981 instruct membar_release() %{
7982 match(MemBarRelease);
7983 ins_cost(VOLATILE_REF_COST);
7984
7985 format %{ "membar_release\n\t"
7986 "dmb ishst\n\tdmb ishld" %}
7987
7988 ins_encode %{
7989 __ block_comment("membar_release");
7990 // These will be merged if AlwaysMergeDMB is enabled.
7991 __ membar(Assembler::StoreStore);
7992 __ membar(Assembler::LoadStore);
7993 %}
7994 ins_pipe(pipe_serial);
7995 %}
7996
7997 instruct membar_storestore() %{
7998 match(MemBarStoreStore);
7999 match(StoreStoreFence);
8000 ins_cost(VOLATILE_REF_COST);
8001
8002 format %{ "MEMBAR-store-store" %}
8003
8004 ins_encode %{
8005 __ membar(Assembler::StoreStore);
8006 %}
8007 ins_pipe(pipe_serial);
8008 %}
8009
8010 instruct membar_release_lock() %{
8011 match(MemBarReleaseLock);
8012 ins_cost(VOLATILE_REF_COST);
8013
8014 format %{ "membar_release_lock (elided)" %}
8015
8016 ins_encode %{
8017 __ block_comment("membar_release_lock (elided)");
8018 %}
8019
8020 ins_pipe(pipe_serial);
8021 %}
8022
8023 instruct membar_storeload() %{
8024 match(MemBarStoreLoad);
8025 ins_cost(VOLATILE_REF_COST*100);
8026
8027 format %{ "MEMBAR-store-load\n\t"
8028 "dmb ish" %}
8029
8030 ins_encode %{
8031 __ block_comment("membar_storeload");
8032 __ membar(Assembler::StoreLoad);
8033 %}
8034
8035 ins_pipe(pipe_serial);
8036 %}
8037
8038 instruct unnecessary_membar_volatile() %{
8039 predicate(unnecessary_volatile(n));
8040 match(MemBarVolatile);
8041 ins_cost(0);
8042
8043 format %{ "membar_volatile (elided)" %}
8044
8045 ins_encode %{
8046 __ block_comment("membar_volatile (elided)");
8047 %}
8048
8049 ins_pipe(pipe_serial);
8050 %}
8051
8052 instruct membar_volatile() %{
8053 match(MemBarVolatile);
8054 ins_cost(VOLATILE_REF_COST*100);
8055
8056 format %{ "membar_volatile\n\t"
8057 "dmb ish"%}
8058
8059 ins_encode %{
8060 __ block_comment("membar_volatile");
8061 __ membar(Assembler::StoreLoad);
8062 %}
8063
8064 ins_pipe(pipe_serial);
8065 %}
8066
8067 instruct membar_full() %{
8068 match(MemBarFull);
8069 ins_cost(VOLATILE_REF_COST*100);
8070
8071 format %{ "membar_full\n\t"
8072 "dmb ish" %}
8073 ins_encode %{
8074 __ block_comment("membar_full");
8075 __ membar(Assembler::AnyAny);
8076 %}
8077
8078 ins_pipe(pipe_serial);
8079 %}
8080
8081 // ============================================================================
8082 // Cast/Convert Instructions
8083
8084 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8085 match(Set dst (CastX2P src));
8086
8087 ins_cost(INSN_COST);
8088 format %{ "mov $dst, $src\t# long -> ptr" %}
8089
8090 ins_encode %{
8091 if ($dst$$reg != $src$$reg) {
8092 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8093 }
8094 %}
8095
8096 ins_pipe(ialu_reg);
8097 %}
8098
8099 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8100 match(Set dst (CastP2X src));
8101
8102 ins_cost(INSN_COST);
8103 format %{ "mov $dst, $src\t# ptr -> long" %}
8104
8105 ins_encode %{
8106 if ($dst$$reg != $src$$reg) {
8107 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8108 }
8109 %}
8110
8111 ins_pipe(ialu_reg);
8112 %}
8113
8114 // Convert oop into int for vectors alignment masking
8115 instruct convP2I(iRegINoSp dst, iRegP src) %{
8116 match(Set dst (ConvL2I (CastP2X src)));
8117
8118 ins_cost(INSN_COST);
8119 format %{ "movw $dst, $src\t# ptr -> int" %}
8120 ins_encode %{
8121 __ movw($dst$$Register, $src$$Register);
8122 %}
8123
8124 ins_pipe(ialu_reg);
8125 %}
8126
8127 // Convert compressed oop into int for vectors alignment masking
8128 // in case of 32bit oops (heap < 4Gb).
8129 instruct convN2I(iRegINoSp dst, iRegN src)
8130 %{
8131 predicate(CompressedOops::shift() == 0);
8132 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8133
8134 ins_cost(INSN_COST);
8135 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8136 ins_encode %{
8137 __ movw($dst$$Register, $src$$Register);
8138 %}
8139
8140 ins_pipe(ialu_reg);
8141 %}
8142
8143
8144 // Convert oop pointer into compressed form
8145 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8146 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8147 match(Set dst (EncodeP src));
8148 effect(KILL cr);
8149 ins_cost(INSN_COST * 3);
8150 format %{ "encode_heap_oop $dst, $src" %}
8151 ins_encode %{
8152 Register s = $src$$Register;
8153 Register d = $dst$$Register;
8154 __ encode_heap_oop(d, s);
8155 %}
8156 ins_pipe(ialu_reg);
8157 %}
8158
8159 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8160 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8161 match(Set dst (EncodeP src));
8162 ins_cost(INSN_COST * 3);
8163 format %{ "encode_heap_oop_not_null $dst, $src" %}
8164 ins_encode %{
8165 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8166 %}
8167 ins_pipe(ialu_reg);
8168 %}
8169
8170 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8171 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8172 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8173 match(Set dst (DecodeN src));
8174 ins_cost(INSN_COST * 3);
8175 format %{ "decode_heap_oop $dst, $src" %}
8176 ins_encode %{
8177 Register s = $src$$Register;
8178 Register d = $dst$$Register;
8179 __ decode_heap_oop(d, s);
8180 %}
8181 ins_pipe(ialu_reg);
8182 %}
8183
8184 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8185 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8186 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8187 match(Set dst (DecodeN src));
8188 ins_cost(INSN_COST * 3);
8189 format %{ "decode_heap_oop_not_null $dst, $src" %}
8190 ins_encode %{
8191 Register s = $src$$Register;
8192 Register d = $dst$$Register;
8193 __ decode_heap_oop_not_null(d, s);
8194 %}
8195 ins_pipe(ialu_reg);
8196 %}
8197
8198 // n.b. AArch64 implementations of encode_klass_not_null and
8199 // decode_klass_not_null do not modify the flags register so, unlike
8200 // Intel, we don't kill CR as a side effect here
8201
8202 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8203 match(Set dst (EncodePKlass src));
8204
8205 ins_cost(INSN_COST * 3);
8206 format %{ "encode_klass_not_null $dst,$src" %}
8207
8208 ins_encode %{
8209 Register src_reg = as_Register($src$$reg);
8210 Register dst_reg = as_Register($dst$$reg);
8211 __ encode_klass_not_null(dst_reg, src_reg);
8212 %}
8213
8214 ins_pipe(ialu_reg);
8215 %}
8216
8217 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8218 match(Set dst (DecodeNKlass src));
8219
8220 ins_cost(INSN_COST * 3);
8221 format %{ "decode_klass_not_null $dst,$src" %}
8222
8223 ins_encode %{
8224 Register src_reg = as_Register($src$$reg);
8225 Register dst_reg = as_Register($dst$$reg);
8226 if (dst_reg != src_reg) {
8227 __ decode_klass_not_null(dst_reg, src_reg);
8228 } else {
8229 __ decode_klass_not_null(dst_reg);
8230 }
8231 %}
8232
8233 ins_pipe(ialu_reg);
8234 %}
8235
8236 instruct checkCastPP(iRegPNoSp dst)
8237 %{
8238 match(Set dst (CheckCastPP dst));
8239
8240 size(0);
8241 format %{ "# checkcastPP of $dst" %}
8242 ins_encode(/* empty encoding */);
8243 ins_pipe(pipe_class_empty);
8244 %}
8245
8246 instruct castPP(iRegPNoSp dst)
8247 %{
8248 match(Set dst (CastPP dst));
8249
8250 size(0);
8251 format %{ "# castPP of $dst" %}
8252 ins_encode(/* empty encoding */);
8253 ins_pipe(pipe_class_empty);
8254 %}
8255
8256 instruct castII(iRegI dst)
8257 %{
8258 predicate(VerifyConstraintCasts == 0);
8259 match(Set dst (CastII dst));
8260
8261 size(0);
8262 format %{ "# castII of $dst" %}
8263 ins_encode(/* empty encoding */);
8264 ins_cost(0);
8265 ins_pipe(pipe_class_empty);
8266 %}
8267
8268 instruct castII_checked(iRegI dst, rFlagsReg cr)
8269 %{
8270 predicate(VerifyConstraintCasts > 0);
8271 match(Set dst (CastII dst));
8272 effect(KILL cr);
8273
8274 format %{ "# castII_checked of $dst" %}
8275 ins_encode %{
8276 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8277 %}
8278 ins_pipe(pipe_slow);
8279 %}
8280
8281 instruct castLL(iRegL dst)
8282 %{
8283 predicate(VerifyConstraintCasts == 0);
8284 match(Set dst (CastLL dst));
8285
8286 size(0);
8287 format %{ "# castLL of $dst" %}
8288 ins_encode(/* empty encoding */);
8289 ins_cost(0);
8290 ins_pipe(pipe_class_empty);
8291 %}
8292
8293 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8294 %{
8295 predicate(VerifyConstraintCasts > 0);
8296 match(Set dst (CastLL dst));
8297 effect(KILL cr);
8298
8299 format %{ "# castLL_checked of $dst" %}
8300 ins_encode %{
8301 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8302 %}
8303 ins_pipe(pipe_slow);
8304 %}
8305
8306 instruct castHH(vRegF dst)
8307 %{
8308 match(Set dst (CastHH dst));
8309 size(0);
8310 format %{ "# castHH of $dst" %}
8311 ins_encode(/* empty encoding */);
8312 ins_cost(0);
8313 ins_pipe(pipe_class_empty);
8314 %}
8315
8316 instruct castFF(vRegF dst)
8317 %{
8318 match(Set dst (CastFF dst));
8319
8320 size(0);
8321 format %{ "# castFF of $dst" %}
8322 ins_encode(/* empty encoding */);
8323 ins_cost(0);
8324 ins_pipe(pipe_class_empty);
8325 %}
8326
8327 instruct castDD(vRegD dst)
8328 %{
8329 match(Set dst (CastDD dst));
8330
8331 size(0);
8332 format %{ "# castDD of $dst" %}
8333 ins_encode(/* empty encoding */);
8334 ins_cost(0);
8335 ins_pipe(pipe_class_empty);
8336 %}
8337
8338 instruct castVV(vReg dst)
8339 %{
8340 match(Set dst (CastVV dst));
8341
8342 size(0);
8343 format %{ "# castVV of $dst" %}
8344 ins_encode(/* empty encoding */);
8345 ins_cost(0);
8346 ins_pipe(pipe_class_empty);
8347 %}
8348
8349 instruct castVVMask(pRegGov dst)
8350 %{
8351 match(Set dst (CastVV dst));
8352
8353 size(0);
8354 format %{ "# castVV of $dst" %}
8355 ins_encode(/* empty encoding */);
8356 ins_cost(0);
8357 ins_pipe(pipe_class_empty);
8358 %}
8359
8360 // Manifest a CmpU result in an integer register.
8361 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8362 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8363 %{
8364 match(Set dst (CmpU3 src1 src2));
8365 effect(KILL flags);
8366
8367 ins_cost(INSN_COST * 3);
8368 format %{
8369 "cmpw $src1, $src2\n\t"
8370 "csetw $dst, ne\n\t"
8371 "cnegw $dst, lo\t# CmpU3(reg)"
8372 %}
8373 ins_encode %{
8374 __ cmpw($src1$$Register, $src2$$Register);
8375 __ csetw($dst$$Register, Assembler::NE);
8376 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8377 %}
8378
8379 ins_pipe(pipe_class_default);
8380 %}
8381
8382 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8383 %{
8384 match(Set dst (CmpU3 src1 src2));
8385 effect(KILL flags);
8386
8387 ins_cost(INSN_COST * 3);
8388 format %{
8389 "subsw zr, $src1, $src2\n\t"
8390 "csetw $dst, ne\n\t"
8391 "cnegw $dst, lo\t# CmpU3(imm)"
8392 %}
8393 ins_encode %{
8394 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8395 __ csetw($dst$$Register, Assembler::NE);
8396 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8397 %}
8398
8399 ins_pipe(pipe_class_default);
8400 %}
8401
8402 // Manifest a CmpUL result in an integer register.
8403 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8404 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8405 %{
8406 match(Set dst (CmpUL3 src1 src2));
8407 effect(KILL flags);
8408
8409 ins_cost(INSN_COST * 3);
8410 format %{
8411 "cmp $src1, $src2\n\t"
8412 "csetw $dst, ne\n\t"
8413 "cnegw $dst, lo\t# CmpUL3(reg)"
8414 %}
8415 ins_encode %{
8416 __ cmp($src1$$Register, $src2$$Register);
8417 __ csetw($dst$$Register, Assembler::NE);
8418 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8419 %}
8420
8421 ins_pipe(pipe_class_default);
8422 %}
8423
8424 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8425 %{
8426 match(Set dst (CmpUL3 src1 src2));
8427 effect(KILL flags);
8428
8429 ins_cost(INSN_COST * 3);
8430 format %{
8431 "subs zr, $src1, $src2\n\t"
8432 "csetw $dst, ne\n\t"
8433 "cnegw $dst, lo\t# CmpUL3(imm)"
8434 %}
8435 ins_encode %{
8436 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8437 __ csetw($dst$$Register, Assembler::NE);
8438 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8439 %}
8440
8441 ins_pipe(pipe_class_default);
8442 %}
8443
8444 // Manifest a CmpL result in an integer register.
8445 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8446 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8447 %{
8448 match(Set dst (CmpL3 src1 src2));
8449 effect(KILL flags);
8450
8451 ins_cost(INSN_COST * 3);
8452 format %{
8453 "cmp $src1, $src2\n\t"
8454 "csetw $dst, ne\n\t"
8455 "cnegw $dst, lt\t# CmpL3(reg)"
8456 %}
8457 ins_encode %{
8458 __ cmp($src1$$Register, $src2$$Register);
8459 __ csetw($dst$$Register, Assembler::NE);
8460 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8461 %}
8462
8463 ins_pipe(pipe_class_default);
8464 %}
8465
8466 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8467 %{
8468 match(Set dst (CmpL3 src1 src2));
8469 effect(KILL flags);
8470
8471 ins_cost(INSN_COST * 3);
8472 format %{
8473 "subs zr, $src1, $src2\n\t"
8474 "csetw $dst, ne\n\t"
8475 "cnegw $dst, lt\t# CmpL3(imm)"
8476 %}
8477 ins_encode %{
8478 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8479 __ csetw($dst$$Register, Assembler::NE);
8480 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8481 %}
8482
8483 ins_pipe(pipe_class_default);
8484 %}
8485
8486 // ============================================================================
8487 // Conditional Move Instructions
8488
8489 // n.b. we have identical rules for both a signed compare op (cmpOp)
8490 // and an unsigned compare op (cmpOpU). it would be nice if we could
8491 // define an op class which merged both inputs and use it to type the
8492 // argument to a single rule. unfortunatelyt his fails because the
8493 // opclass does not live up to the COND_INTER interface of its
8494 // component operands. When the generic code tries to negate the
8495 // operand it ends up running the generci Machoper::negate method
8496 // which throws a ShouldNotHappen. So, we have to provide two flavours
8497 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8498
8499 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8500 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8501
8502 ins_cost(INSN_COST * 2);
8503 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8504
8505 ins_encode %{
8506 __ cselw(as_Register($dst$$reg),
8507 as_Register($src2$$reg),
8508 as_Register($src1$$reg),
8509 (Assembler::Condition)$cmp$$cmpcode);
8510 %}
8511
8512 ins_pipe(icond_reg_reg);
8513 %}
8514
8515 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8516 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8517
8518 ins_cost(INSN_COST * 2);
8519 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8520
8521 ins_encode %{
8522 __ cselw(as_Register($dst$$reg),
8523 as_Register($src2$$reg),
8524 as_Register($src1$$reg),
8525 (Assembler::Condition)$cmp$$cmpcode);
8526 %}
8527
8528 ins_pipe(icond_reg_reg);
8529 %}
8530
8531 // special cases where one arg is zero
8532
8533 // n.b. this is selected in preference to the rule above because it
8534 // avoids loading constant 0 into a source register
8535
8536 // TODO
8537 // we ought only to be able to cull one of these variants as the ideal
8538 // transforms ought always to order the zero consistently (to left/right?)
8539
8540 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8541 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8542
8543 ins_cost(INSN_COST * 2);
8544 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8545
8546 ins_encode %{
8547 __ cselw(as_Register($dst$$reg),
8548 as_Register($src$$reg),
8549 zr,
8550 (Assembler::Condition)$cmp$$cmpcode);
8551 %}
8552
8553 ins_pipe(icond_reg);
8554 %}
8555
8556 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8557 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8558
8559 ins_cost(INSN_COST * 2);
8560 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8561
8562 ins_encode %{
8563 __ cselw(as_Register($dst$$reg),
8564 as_Register($src$$reg),
8565 zr,
8566 (Assembler::Condition)$cmp$$cmpcode);
8567 %}
8568
8569 ins_pipe(icond_reg);
8570 %}
8571
8572 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8573 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8574
8575 ins_cost(INSN_COST * 2);
8576 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8577
8578 ins_encode %{
8579 __ cselw(as_Register($dst$$reg),
8580 zr,
8581 as_Register($src$$reg),
8582 (Assembler::Condition)$cmp$$cmpcode);
8583 %}
8584
8585 ins_pipe(icond_reg);
8586 %}
8587
8588 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8589 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8590
8591 ins_cost(INSN_COST * 2);
8592 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8593
8594 ins_encode %{
8595 __ cselw(as_Register($dst$$reg),
8596 zr,
8597 as_Register($src$$reg),
8598 (Assembler::Condition)$cmp$$cmpcode);
8599 %}
8600
8601 ins_pipe(icond_reg);
8602 %}
8603
8604 // special case for creating a boolean 0 or 1
8605
8606 // n.b. this is selected in preference to the rule above because it
8607 // avoids loading constants 0 and 1 into a source register
8608
8609 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8610 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8611
8612 ins_cost(INSN_COST * 2);
8613 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8614
8615 ins_encode %{
8616 // equivalently
8617 // cset(as_Register($dst$$reg),
8618 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8619 __ csincw(as_Register($dst$$reg),
8620 zr,
8621 zr,
8622 (Assembler::Condition)$cmp$$cmpcode);
8623 %}
8624
8625 ins_pipe(icond_none);
8626 %}
8627
8628 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8629 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8630
8631 ins_cost(INSN_COST * 2);
8632 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8633
8634 ins_encode %{
8635 // equivalently
8636 // cset(as_Register($dst$$reg),
8637 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8638 __ csincw(as_Register($dst$$reg),
8639 zr,
8640 zr,
8641 (Assembler::Condition)$cmp$$cmpcode);
8642 %}
8643
8644 ins_pipe(icond_none);
8645 %}
8646
8647 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8648 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8649
8650 ins_cost(INSN_COST * 2);
8651 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8652
8653 ins_encode %{
8654 __ csel(as_Register($dst$$reg),
8655 as_Register($src2$$reg),
8656 as_Register($src1$$reg),
8657 (Assembler::Condition)$cmp$$cmpcode);
8658 %}
8659
8660 ins_pipe(icond_reg_reg);
8661 %}
8662
8663 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8664 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8665
8666 ins_cost(INSN_COST * 2);
8667 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8668
8669 ins_encode %{
8670 __ csel(as_Register($dst$$reg),
8671 as_Register($src2$$reg),
8672 as_Register($src1$$reg),
8673 (Assembler::Condition)$cmp$$cmpcode);
8674 %}
8675
8676 ins_pipe(icond_reg_reg);
8677 %}
8678
8679 // special cases where one arg is zero
8680
8681 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8682 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8683
8684 ins_cost(INSN_COST * 2);
8685 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8686
8687 ins_encode %{
8688 __ csel(as_Register($dst$$reg),
8689 zr,
8690 as_Register($src$$reg),
8691 (Assembler::Condition)$cmp$$cmpcode);
8692 %}
8693
8694 ins_pipe(icond_reg);
8695 %}
8696
8697 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8698 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8699
8700 ins_cost(INSN_COST * 2);
8701 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8702
8703 ins_encode %{
8704 __ csel(as_Register($dst$$reg),
8705 zr,
8706 as_Register($src$$reg),
8707 (Assembler::Condition)$cmp$$cmpcode);
8708 %}
8709
8710 ins_pipe(icond_reg);
8711 %}
8712
8713 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8714 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8715
8716 ins_cost(INSN_COST * 2);
8717 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8718
8719 ins_encode %{
8720 __ csel(as_Register($dst$$reg),
8721 as_Register($src$$reg),
8722 zr,
8723 (Assembler::Condition)$cmp$$cmpcode);
8724 %}
8725
8726 ins_pipe(icond_reg);
8727 %}
8728
8729 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8730 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8731
8732 ins_cost(INSN_COST * 2);
8733 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8734
8735 ins_encode %{
8736 __ csel(as_Register($dst$$reg),
8737 as_Register($src$$reg),
8738 zr,
8739 (Assembler::Condition)$cmp$$cmpcode);
8740 %}
8741
8742 ins_pipe(icond_reg);
8743 %}
8744
8745 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8746 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8747
8748 ins_cost(INSN_COST * 2);
8749 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8750
8751 ins_encode %{
8752 __ csel(as_Register($dst$$reg),
8753 as_Register($src2$$reg),
8754 as_Register($src1$$reg),
8755 (Assembler::Condition)$cmp$$cmpcode);
8756 %}
8757
8758 ins_pipe(icond_reg_reg);
8759 %}
8760
8761 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8762 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8763
8764 ins_cost(INSN_COST * 2);
8765 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8766
8767 ins_encode %{
8768 __ csel(as_Register($dst$$reg),
8769 as_Register($src2$$reg),
8770 as_Register($src1$$reg),
8771 (Assembler::Condition)$cmp$$cmpcode);
8772 %}
8773
8774 ins_pipe(icond_reg_reg);
8775 %}
8776
8777 // special cases where one arg is zero
8778
8779 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8780 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8781
8782 ins_cost(INSN_COST * 2);
8783 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8784
8785 ins_encode %{
8786 __ csel(as_Register($dst$$reg),
8787 zr,
8788 as_Register($src$$reg),
8789 (Assembler::Condition)$cmp$$cmpcode);
8790 %}
8791
8792 ins_pipe(icond_reg);
8793 %}
8794
8795 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8796 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8797
8798 ins_cost(INSN_COST * 2);
8799 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8800
8801 ins_encode %{
8802 __ csel(as_Register($dst$$reg),
8803 zr,
8804 as_Register($src$$reg),
8805 (Assembler::Condition)$cmp$$cmpcode);
8806 %}
8807
8808 ins_pipe(icond_reg);
8809 %}
8810
8811 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8812 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8813
8814 ins_cost(INSN_COST * 2);
8815 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8816
8817 ins_encode %{
8818 __ csel(as_Register($dst$$reg),
8819 as_Register($src$$reg),
8820 zr,
8821 (Assembler::Condition)$cmp$$cmpcode);
8822 %}
8823
8824 ins_pipe(icond_reg);
8825 %}
8826
8827 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8828 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8829
8830 ins_cost(INSN_COST * 2);
8831 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8832
8833 ins_encode %{
8834 __ csel(as_Register($dst$$reg),
8835 as_Register($src$$reg),
8836 zr,
8837 (Assembler::Condition)$cmp$$cmpcode);
8838 %}
8839
8840 ins_pipe(icond_reg);
8841 %}
8842
8843 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8844 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8845
8846 ins_cost(INSN_COST * 2);
8847 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8848
8849 ins_encode %{
8850 __ cselw(as_Register($dst$$reg),
8851 as_Register($src2$$reg),
8852 as_Register($src1$$reg),
8853 (Assembler::Condition)$cmp$$cmpcode);
8854 %}
8855
8856 ins_pipe(icond_reg_reg);
8857 %}
8858
8859 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8860 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8861
8862 ins_cost(INSN_COST * 2);
8863 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8864
8865 ins_encode %{
8866 __ cselw(as_Register($dst$$reg),
8867 as_Register($src2$$reg),
8868 as_Register($src1$$reg),
8869 (Assembler::Condition)$cmp$$cmpcode);
8870 %}
8871
8872 ins_pipe(icond_reg_reg);
8873 %}
8874
8875 // special cases where one arg is zero
8876
8877 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8878 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8879
8880 ins_cost(INSN_COST * 2);
8881 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8882
8883 ins_encode %{
8884 __ cselw(as_Register($dst$$reg),
8885 zr,
8886 as_Register($src$$reg),
8887 (Assembler::Condition)$cmp$$cmpcode);
8888 %}
8889
8890 ins_pipe(icond_reg);
8891 %}
8892
8893 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8894 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8895
8896 ins_cost(INSN_COST * 2);
8897 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8898
8899 ins_encode %{
8900 __ cselw(as_Register($dst$$reg),
8901 zr,
8902 as_Register($src$$reg),
8903 (Assembler::Condition)$cmp$$cmpcode);
8904 %}
8905
8906 ins_pipe(icond_reg);
8907 %}
8908
8909 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8910 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8911
8912 ins_cost(INSN_COST * 2);
8913 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8914
8915 ins_encode %{
8916 __ cselw(as_Register($dst$$reg),
8917 as_Register($src$$reg),
8918 zr,
8919 (Assembler::Condition)$cmp$$cmpcode);
8920 %}
8921
8922 ins_pipe(icond_reg);
8923 %}
8924
8925 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8926 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8927
8928 ins_cost(INSN_COST * 2);
8929 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8930
8931 ins_encode %{
8932 __ cselw(as_Register($dst$$reg),
8933 as_Register($src$$reg),
8934 zr,
8935 (Assembler::Condition)$cmp$$cmpcode);
8936 %}
8937
8938 ins_pipe(icond_reg);
8939 %}
8940
8941 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8942 %{
8943 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8944
8945 ins_cost(INSN_COST * 3);
8946
8947 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8948 ins_encode %{
8949 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8950 __ fcsels(as_FloatRegister($dst$$reg),
8951 as_FloatRegister($src2$$reg),
8952 as_FloatRegister($src1$$reg),
8953 cond);
8954 %}
8955
8956 ins_pipe(fp_cond_reg_reg_s);
8957 %}
8958
8959 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8960 %{
8961 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8962
8963 ins_cost(INSN_COST * 3);
8964
8965 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8966 ins_encode %{
8967 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8968 __ fcsels(as_FloatRegister($dst$$reg),
8969 as_FloatRegister($src2$$reg),
8970 as_FloatRegister($src1$$reg),
8971 cond);
8972 %}
8973
8974 ins_pipe(fp_cond_reg_reg_s);
8975 %}
8976
8977 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8978 %{
8979 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8980
8981 ins_cost(INSN_COST * 3);
8982
8983 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8984 ins_encode %{
8985 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8986 __ fcseld(as_FloatRegister($dst$$reg),
8987 as_FloatRegister($src2$$reg),
8988 as_FloatRegister($src1$$reg),
8989 cond);
8990 %}
8991
8992 ins_pipe(fp_cond_reg_reg_d);
8993 %}
8994
8995 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
8996 %{
8997 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8998
8999 ins_cost(INSN_COST * 3);
9000
9001 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9002 ins_encode %{
9003 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9004 __ fcseld(as_FloatRegister($dst$$reg),
9005 as_FloatRegister($src2$$reg),
9006 as_FloatRegister($src1$$reg),
9007 cond);
9008 %}
9009
9010 ins_pipe(fp_cond_reg_reg_d);
9011 %}
9012
9013 // ============================================================================
9014 // Arithmetic Instructions
9015 //
9016
9017 // Integer Addition
9018
9019 // TODO
9020 // these currently employ operations which do not set CR and hence are
9021 // not flagged as killing CR but we would like to isolate the cases
9022 // where we want to set flags from those where we don't. need to work
9023 // out how to do that.
9024
9025 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9026 match(Set dst (AddI src1 src2));
9027
9028 ins_cost(INSN_COST);
9029 format %{ "addw $dst, $src1, $src2" %}
9030
9031 ins_encode %{
9032 __ addw(as_Register($dst$$reg),
9033 as_Register($src1$$reg),
9034 as_Register($src2$$reg));
9035 %}
9036
9037 ins_pipe(ialu_reg_reg);
9038 %}
9039
9040 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9041 match(Set dst (AddI src1 src2));
9042
9043 ins_cost(INSN_COST);
9044 format %{ "addw $dst, $src1, $src2" %}
9045
9046 // use opcode to indicate that this is an add not a sub
9047 opcode(0x0);
9048
9049 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9050
9051 ins_pipe(ialu_reg_imm);
9052 %}
9053
9054 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9055 match(Set dst (AddI (ConvL2I src1) src2));
9056
9057 ins_cost(INSN_COST);
9058 format %{ "addw $dst, $src1, $src2" %}
9059
9060 // use opcode to indicate that this is an add not a sub
9061 opcode(0x0);
9062
9063 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9064
9065 ins_pipe(ialu_reg_imm);
9066 %}
9067
9068 // Pointer Addition
9069 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9070 match(Set dst (AddP src1 src2));
9071
9072 ins_cost(INSN_COST);
9073 format %{ "add $dst, $src1, $src2\t# ptr" %}
9074
9075 ins_encode %{
9076 __ add(as_Register($dst$$reg),
9077 as_Register($src1$$reg),
9078 as_Register($src2$$reg));
9079 %}
9080
9081 ins_pipe(ialu_reg_reg);
9082 %}
9083
9084 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9085 match(Set dst (AddP src1 (ConvI2L src2)));
9086
9087 ins_cost(1.9 * INSN_COST);
9088 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9089
9090 ins_encode %{
9091 __ add(as_Register($dst$$reg),
9092 as_Register($src1$$reg),
9093 as_Register($src2$$reg), ext::sxtw);
9094 %}
9095
9096 ins_pipe(ialu_reg_reg);
9097 %}
9098
9099 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9100 match(Set dst (AddP src1 (LShiftL src2 scale)));
9101
9102 ins_cost(1.9 * INSN_COST);
9103 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9104
9105 ins_encode %{
9106 __ lea(as_Register($dst$$reg),
9107 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9108 Address::lsl($scale$$constant)));
9109 %}
9110
9111 ins_pipe(ialu_reg_reg_shift);
9112 %}
9113
9114 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9115 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9116
9117 ins_cost(1.9 * INSN_COST);
9118 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9119
9120 ins_encode %{
9121 __ lea(as_Register($dst$$reg),
9122 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9123 Address::sxtw($scale$$constant)));
9124 %}
9125
9126 ins_pipe(ialu_reg_reg_shift);
9127 %}
9128
9129 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9130 match(Set dst (LShiftL (ConvI2L src) scale));
9131
9132 ins_cost(INSN_COST);
9133 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9134
9135 ins_encode %{
9136 __ sbfiz(as_Register($dst$$reg),
9137 as_Register($src$$reg),
9138 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9139 %}
9140
9141 ins_pipe(ialu_reg_shift);
9142 %}
9143
9144 // Pointer Immediate Addition
9145 // n.b. this needs to be more expensive than using an indirect memory
9146 // operand
9147 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9148 match(Set dst (AddP src1 src2));
9149
9150 ins_cost(INSN_COST);
9151 format %{ "add $dst, $src1, $src2\t# ptr" %}
9152
9153 // use opcode to indicate that this is an add not a sub
9154 opcode(0x0);
9155
9156 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9157
9158 ins_pipe(ialu_reg_imm);
9159 %}
9160
9161 // Long Addition
9162 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9163
9164 match(Set dst (AddL src1 src2));
9165
9166 ins_cost(INSN_COST);
9167 format %{ "add $dst, $src1, $src2" %}
9168
9169 ins_encode %{
9170 __ add(as_Register($dst$$reg),
9171 as_Register($src1$$reg),
9172 as_Register($src2$$reg));
9173 %}
9174
9175 ins_pipe(ialu_reg_reg);
9176 %}
9177
9178 // No constant pool entries requiredLong Immediate Addition.
9179 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9180 match(Set dst (AddL src1 src2));
9181
9182 ins_cost(INSN_COST);
9183 format %{ "add $dst, $src1, $src2" %}
9184
9185 // use opcode to indicate that this is an add not a sub
9186 opcode(0x0);
9187
9188 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9189
9190 ins_pipe(ialu_reg_imm);
9191 %}
9192
9193 // Integer Subtraction
9194 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9195 match(Set dst (SubI src1 src2));
9196
9197 ins_cost(INSN_COST);
9198 format %{ "subw $dst, $src1, $src2" %}
9199
9200 ins_encode %{
9201 __ subw(as_Register($dst$$reg),
9202 as_Register($src1$$reg),
9203 as_Register($src2$$reg));
9204 %}
9205
9206 ins_pipe(ialu_reg_reg);
9207 %}
9208
9209 // Immediate Subtraction
9210 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9211 match(Set dst (SubI src1 src2));
9212
9213 ins_cost(INSN_COST);
9214 format %{ "subw $dst, $src1, $src2" %}
9215
9216 // use opcode to indicate that this is a sub not an add
9217 opcode(0x1);
9218
9219 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9220
9221 ins_pipe(ialu_reg_imm);
9222 %}
9223
9224 // Long Subtraction
9225 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9226
9227 match(Set dst (SubL src1 src2));
9228
9229 ins_cost(INSN_COST);
9230 format %{ "sub $dst, $src1, $src2" %}
9231
9232 ins_encode %{
9233 __ sub(as_Register($dst$$reg),
9234 as_Register($src1$$reg),
9235 as_Register($src2$$reg));
9236 %}
9237
9238 ins_pipe(ialu_reg_reg);
9239 %}
9240
9241 // No constant pool entries requiredLong Immediate Subtraction.
9242 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9243 match(Set dst (SubL src1 src2));
9244
9245 ins_cost(INSN_COST);
9246 format %{ "sub$dst, $src1, $src2" %}
9247
9248 // use opcode to indicate that this is a sub not an add
9249 opcode(0x1);
9250
9251 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9252
9253 ins_pipe(ialu_reg_imm);
9254 %}
9255
9256 // Integer Negation (special case for sub)
9257
9258 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9259 match(Set dst (SubI zero src));
9260
9261 ins_cost(INSN_COST);
9262 format %{ "negw $dst, $src\t# int" %}
9263
9264 ins_encode %{
9265 __ negw(as_Register($dst$$reg),
9266 as_Register($src$$reg));
9267 %}
9268
9269 ins_pipe(ialu_reg);
9270 %}
9271
9272 // Long Negation
9273
9274 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9275 match(Set dst (SubL zero src));
9276
9277 ins_cost(INSN_COST);
9278 format %{ "neg $dst, $src\t# long" %}
9279
9280 ins_encode %{
9281 __ neg(as_Register($dst$$reg),
9282 as_Register($src$$reg));
9283 %}
9284
9285 ins_pipe(ialu_reg);
9286 %}
9287
9288 // Integer Multiply
9289
9290 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9291 match(Set dst (MulI src1 src2));
9292
9293 ins_cost(INSN_COST * 3);
9294 format %{ "mulw $dst, $src1, $src2" %}
9295
9296 ins_encode %{
9297 __ mulw(as_Register($dst$$reg),
9298 as_Register($src1$$reg),
9299 as_Register($src2$$reg));
9300 %}
9301
9302 ins_pipe(imul_reg_reg);
9303 %}
9304
9305 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9306 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9307
9308 ins_cost(INSN_COST * 3);
9309 format %{ "smull $dst, $src1, $src2" %}
9310
9311 ins_encode %{
9312 __ smull(as_Register($dst$$reg),
9313 as_Register($src1$$reg),
9314 as_Register($src2$$reg));
9315 %}
9316
9317 ins_pipe(imul_reg_reg);
9318 %}
9319
9320 // Long Multiply
9321
9322 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9323 match(Set dst (MulL src1 src2));
9324
9325 ins_cost(INSN_COST * 5);
9326 format %{ "mul $dst, $src1, $src2" %}
9327
9328 ins_encode %{
9329 __ mul(as_Register($dst$$reg),
9330 as_Register($src1$$reg),
9331 as_Register($src2$$reg));
9332 %}
9333
9334 ins_pipe(lmul_reg_reg);
9335 %}
9336
9337 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9338 %{
9339 match(Set dst (MulHiL src1 src2));
9340
9341 ins_cost(INSN_COST * 7);
9342 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9343
9344 ins_encode %{
9345 __ smulh(as_Register($dst$$reg),
9346 as_Register($src1$$reg),
9347 as_Register($src2$$reg));
9348 %}
9349
9350 ins_pipe(lmul_reg_reg);
9351 %}
9352
9353 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9354 %{
9355 match(Set dst (UMulHiL src1 src2));
9356
9357 ins_cost(INSN_COST * 7);
9358 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9359
9360 ins_encode %{
9361 __ umulh(as_Register($dst$$reg),
9362 as_Register($src1$$reg),
9363 as_Register($src2$$reg));
9364 %}
9365
9366 ins_pipe(lmul_reg_reg);
9367 %}
9368
9369 // Combined Integer Multiply & Add/Sub
9370
9371 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9372 match(Set dst (AddI src3 (MulI src1 src2)));
9373
9374 ins_cost(INSN_COST * 3);
9375 format %{ "madd $dst, $src1, $src2, $src3" %}
9376
9377 ins_encode %{
9378 __ maddw(as_Register($dst$$reg),
9379 as_Register($src1$$reg),
9380 as_Register($src2$$reg),
9381 as_Register($src3$$reg));
9382 %}
9383
9384 ins_pipe(imac_reg_reg);
9385 %}
9386
9387 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9388 match(Set dst (SubI src3 (MulI src1 src2)));
9389
9390 ins_cost(INSN_COST * 3);
9391 format %{ "msub $dst, $src1, $src2, $src3" %}
9392
9393 ins_encode %{
9394 __ msubw(as_Register($dst$$reg),
9395 as_Register($src1$$reg),
9396 as_Register($src2$$reg),
9397 as_Register($src3$$reg));
9398 %}
9399
9400 ins_pipe(imac_reg_reg);
9401 %}
9402
9403 // Combined Integer Multiply & Neg
9404
9405 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9406 match(Set dst (MulI (SubI zero src1) src2));
9407
9408 ins_cost(INSN_COST * 3);
9409 format %{ "mneg $dst, $src1, $src2" %}
9410
9411 ins_encode %{
9412 __ mnegw(as_Register($dst$$reg),
9413 as_Register($src1$$reg),
9414 as_Register($src2$$reg));
9415 %}
9416
9417 ins_pipe(imac_reg_reg);
9418 %}
9419
9420 // Combined Long Multiply & Add/Sub
9421
9422 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9423 match(Set dst (AddL src3 (MulL src1 src2)));
9424
9425 ins_cost(INSN_COST * 5);
9426 format %{ "madd $dst, $src1, $src2, $src3" %}
9427
9428 ins_encode %{
9429 __ madd(as_Register($dst$$reg),
9430 as_Register($src1$$reg),
9431 as_Register($src2$$reg),
9432 as_Register($src3$$reg));
9433 %}
9434
9435 ins_pipe(lmac_reg_reg);
9436 %}
9437
9438 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9439 match(Set dst (SubL src3 (MulL src1 src2)));
9440
9441 ins_cost(INSN_COST * 5);
9442 format %{ "msub $dst, $src1, $src2, $src3" %}
9443
9444 ins_encode %{
9445 __ msub(as_Register($dst$$reg),
9446 as_Register($src1$$reg),
9447 as_Register($src2$$reg),
9448 as_Register($src3$$reg));
9449 %}
9450
9451 ins_pipe(lmac_reg_reg);
9452 %}
9453
9454 // Combined Long Multiply & Neg
9455
9456 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9457 match(Set dst (MulL (SubL zero src1) src2));
9458
9459 ins_cost(INSN_COST * 5);
9460 format %{ "mneg $dst, $src1, $src2" %}
9461
9462 ins_encode %{
9463 __ mneg(as_Register($dst$$reg),
9464 as_Register($src1$$reg),
9465 as_Register($src2$$reg));
9466 %}
9467
9468 ins_pipe(lmac_reg_reg);
9469 %}
9470
9471 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9472
9473 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9474 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9475
9476 ins_cost(INSN_COST * 3);
9477 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9478
9479 ins_encode %{
9480 __ smaddl(as_Register($dst$$reg),
9481 as_Register($src1$$reg),
9482 as_Register($src2$$reg),
9483 as_Register($src3$$reg));
9484 %}
9485
9486 ins_pipe(imac_reg_reg);
9487 %}
9488
9489 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9490 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9491
9492 ins_cost(INSN_COST * 3);
9493 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9494
9495 ins_encode %{
9496 __ smsubl(as_Register($dst$$reg),
9497 as_Register($src1$$reg),
9498 as_Register($src2$$reg),
9499 as_Register($src3$$reg));
9500 %}
9501
9502 ins_pipe(imac_reg_reg);
9503 %}
9504
9505 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9506 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9507
9508 ins_cost(INSN_COST * 3);
9509 format %{ "smnegl $dst, $src1, $src2" %}
9510
9511 ins_encode %{
9512 __ smnegl(as_Register($dst$$reg),
9513 as_Register($src1$$reg),
9514 as_Register($src2$$reg));
9515 %}
9516
9517 ins_pipe(imac_reg_reg);
9518 %}
9519
9520 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9521
9522 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9523 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9524
9525 ins_cost(INSN_COST * 5);
9526 format %{ "mulw rscratch1, $src1, $src2\n\t"
9527 "maddw $dst, $src3, $src4, rscratch1" %}
9528
9529 ins_encode %{
9530 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9531 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9532
9533 ins_pipe(imac_reg_reg);
9534 %}
9535
9536 // Integer Divide
9537
9538 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9539 match(Set dst (DivI src1 src2));
9540
9541 ins_cost(INSN_COST * 19);
9542 format %{ "sdivw $dst, $src1, $src2" %}
9543
9544 ins_encode(aarch64_enc_divw(dst, src1, src2));
9545 ins_pipe(idiv_reg_reg);
9546 %}
9547
9548 // Long Divide
9549
9550 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9551 match(Set dst (DivL src1 src2));
9552
9553 ins_cost(INSN_COST * 35);
9554 format %{ "sdiv $dst, $src1, $src2" %}
9555
9556 ins_encode(aarch64_enc_div(dst, src1, src2));
9557 ins_pipe(ldiv_reg_reg);
9558 %}
9559
9560 // Integer Remainder
9561
9562 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9563 match(Set dst (ModI src1 src2));
9564
9565 ins_cost(INSN_COST * 22);
9566 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9567 "msubw $dst, rscratch1, $src2, $src1" %}
9568
9569 ins_encode(aarch64_enc_modw(dst, src1, src2));
9570 ins_pipe(idiv_reg_reg);
9571 %}
9572
9573 // Long Remainder
9574
9575 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9576 match(Set dst (ModL src1 src2));
9577
9578 ins_cost(INSN_COST * 38);
9579 format %{ "sdiv rscratch1, $src1, $src2\n"
9580 "msub $dst, rscratch1, $src2, $src1" %}
9581
9582 ins_encode(aarch64_enc_mod(dst, src1, src2));
9583 ins_pipe(ldiv_reg_reg);
9584 %}
9585
9586 // Unsigned Integer Divide
9587
9588 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9589 match(Set dst (UDivI src1 src2));
9590
9591 ins_cost(INSN_COST * 19);
9592 format %{ "udivw $dst, $src1, $src2" %}
9593
9594 ins_encode %{
9595 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9596 %}
9597
9598 ins_pipe(idiv_reg_reg);
9599 %}
9600
9601 // Unsigned Long Divide
9602
9603 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9604 match(Set dst (UDivL src1 src2));
9605
9606 ins_cost(INSN_COST * 35);
9607 format %{ "udiv $dst, $src1, $src2" %}
9608
9609 ins_encode %{
9610 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9611 %}
9612
9613 ins_pipe(ldiv_reg_reg);
9614 %}
9615
9616 // Unsigned Integer Remainder
9617
9618 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9619 match(Set dst (UModI src1 src2));
9620
9621 ins_cost(INSN_COST * 22);
9622 format %{ "udivw rscratch1, $src1, $src2\n\t"
9623 "msubw $dst, rscratch1, $src2, $src1" %}
9624
9625 ins_encode %{
9626 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9627 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9628 %}
9629
9630 ins_pipe(idiv_reg_reg);
9631 %}
9632
9633 // Unsigned Long Remainder
9634
9635 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9636 match(Set dst (UModL src1 src2));
9637
9638 ins_cost(INSN_COST * 38);
9639 format %{ "udiv rscratch1, $src1, $src2\n"
9640 "msub $dst, rscratch1, $src2, $src1" %}
9641
9642 ins_encode %{
9643 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9644 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9645 %}
9646
9647 ins_pipe(ldiv_reg_reg);
9648 %}
9649
9650 // Integer Shifts
9651
9652 // Shift Left Register
9653 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9654 match(Set dst (LShiftI src1 src2));
9655
9656 ins_cost(INSN_COST * 2);
9657 format %{ "lslvw $dst, $src1, $src2" %}
9658
9659 ins_encode %{
9660 __ lslvw(as_Register($dst$$reg),
9661 as_Register($src1$$reg),
9662 as_Register($src2$$reg));
9663 %}
9664
9665 ins_pipe(ialu_reg_reg_vshift);
9666 %}
9667
9668 // Shift Left Immediate
9669 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9670 match(Set dst (LShiftI src1 src2));
9671
9672 ins_cost(INSN_COST);
9673 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9674
9675 ins_encode %{
9676 __ lslw(as_Register($dst$$reg),
9677 as_Register($src1$$reg),
9678 $src2$$constant & 0x1f);
9679 %}
9680
9681 ins_pipe(ialu_reg_shift);
9682 %}
9683
9684 // Shift Right Logical Register
9685 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9686 match(Set dst (URShiftI src1 src2));
9687
9688 ins_cost(INSN_COST * 2);
9689 format %{ "lsrvw $dst, $src1, $src2" %}
9690
9691 ins_encode %{
9692 __ lsrvw(as_Register($dst$$reg),
9693 as_Register($src1$$reg),
9694 as_Register($src2$$reg));
9695 %}
9696
9697 ins_pipe(ialu_reg_reg_vshift);
9698 %}
9699
9700 // Shift Right Logical Immediate
9701 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9702 match(Set dst (URShiftI src1 src2));
9703
9704 ins_cost(INSN_COST);
9705 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9706
9707 ins_encode %{
9708 __ lsrw(as_Register($dst$$reg),
9709 as_Register($src1$$reg),
9710 $src2$$constant & 0x1f);
9711 %}
9712
9713 ins_pipe(ialu_reg_shift);
9714 %}
9715
9716 // Shift Right Arithmetic Register
9717 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9718 match(Set dst (RShiftI src1 src2));
9719
9720 ins_cost(INSN_COST * 2);
9721 format %{ "asrvw $dst, $src1, $src2" %}
9722
9723 ins_encode %{
9724 __ asrvw(as_Register($dst$$reg),
9725 as_Register($src1$$reg),
9726 as_Register($src2$$reg));
9727 %}
9728
9729 ins_pipe(ialu_reg_reg_vshift);
9730 %}
9731
9732 // Shift Right Arithmetic Immediate
9733 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9734 match(Set dst (RShiftI src1 src2));
9735
9736 ins_cost(INSN_COST);
9737 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9738
9739 ins_encode %{
9740 __ asrw(as_Register($dst$$reg),
9741 as_Register($src1$$reg),
9742 $src2$$constant & 0x1f);
9743 %}
9744
9745 ins_pipe(ialu_reg_shift);
9746 %}
9747
9748 // Combined Int Mask and Right Shift (using UBFM)
9749 // TODO
9750
9751 // Long Shifts
9752
9753 // Shift Left Register
9754 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9755 match(Set dst (LShiftL src1 src2));
9756
9757 ins_cost(INSN_COST * 2);
9758 format %{ "lslv $dst, $src1, $src2" %}
9759
9760 ins_encode %{
9761 __ lslv(as_Register($dst$$reg),
9762 as_Register($src1$$reg),
9763 as_Register($src2$$reg));
9764 %}
9765
9766 ins_pipe(ialu_reg_reg_vshift);
9767 %}
9768
9769 // Shift Left Immediate
9770 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9771 match(Set dst (LShiftL src1 src2));
9772
9773 ins_cost(INSN_COST);
9774 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9775
9776 ins_encode %{
9777 __ lsl(as_Register($dst$$reg),
9778 as_Register($src1$$reg),
9779 $src2$$constant & 0x3f);
9780 %}
9781
9782 ins_pipe(ialu_reg_shift);
9783 %}
9784
9785 // Shift Right Logical Register
9786 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9787 match(Set dst (URShiftL src1 src2));
9788
9789 ins_cost(INSN_COST * 2);
9790 format %{ "lsrv $dst, $src1, $src2" %}
9791
9792 ins_encode %{
9793 __ lsrv(as_Register($dst$$reg),
9794 as_Register($src1$$reg),
9795 as_Register($src2$$reg));
9796 %}
9797
9798 ins_pipe(ialu_reg_reg_vshift);
9799 %}
9800
9801 // Shift Right Logical Immediate
9802 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9803 match(Set dst (URShiftL src1 src2));
9804
9805 ins_cost(INSN_COST);
9806 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9807
9808 ins_encode %{
9809 __ lsr(as_Register($dst$$reg),
9810 as_Register($src1$$reg),
9811 $src2$$constant & 0x3f);
9812 %}
9813
9814 ins_pipe(ialu_reg_shift);
9815 %}
9816
9817 // A special-case pattern for card table stores.
9818 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9819 match(Set dst (URShiftL (CastP2X src1) src2));
9820
9821 ins_cost(INSN_COST);
9822 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9823
9824 ins_encode %{
9825 __ lsr(as_Register($dst$$reg),
9826 as_Register($src1$$reg),
9827 $src2$$constant & 0x3f);
9828 %}
9829
9830 ins_pipe(ialu_reg_shift);
9831 %}
9832
9833 // Shift Right Arithmetic Register
9834 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9835 match(Set dst (RShiftL src1 src2));
9836
9837 ins_cost(INSN_COST * 2);
9838 format %{ "asrv $dst, $src1, $src2" %}
9839
9840 ins_encode %{
9841 __ asrv(as_Register($dst$$reg),
9842 as_Register($src1$$reg),
9843 as_Register($src2$$reg));
9844 %}
9845
9846 ins_pipe(ialu_reg_reg_vshift);
9847 %}
9848
9849 // Shift Right Arithmetic Immediate
9850 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9851 match(Set dst (RShiftL src1 src2));
9852
9853 ins_cost(INSN_COST);
9854 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9855
9856 ins_encode %{
9857 __ asr(as_Register($dst$$reg),
9858 as_Register($src1$$reg),
9859 $src2$$constant & 0x3f);
9860 %}
9861
9862 ins_pipe(ialu_reg_shift);
9863 %}
9864
9865 // BEGIN This section of the file is automatically generated. Do not edit --------------
9866 // This section is generated from aarch64_ad.m4
9867
9868 // This pattern is automatically generated from aarch64_ad.m4
9869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9870 instruct regL_not_reg(iRegLNoSp dst,
9871 iRegL src1, immL_M1 m1,
9872 rFlagsReg cr) %{
9873 match(Set dst (XorL src1 m1));
9874 ins_cost(INSN_COST);
9875 format %{ "eon $dst, $src1, zr" %}
9876
9877 ins_encode %{
9878 __ eon(as_Register($dst$$reg),
9879 as_Register($src1$$reg),
9880 zr,
9881 Assembler::LSL, 0);
9882 %}
9883
9884 ins_pipe(ialu_reg);
9885 %}
9886
9887 // This pattern is automatically generated from aarch64_ad.m4
9888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9889 instruct regI_not_reg(iRegINoSp dst,
9890 iRegIorL2I src1, immI_M1 m1,
9891 rFlagsReg cr) %{
9892 match(Set dst (XorI src1 m1));
9893 ins_cost(INSN_COST);
9894 format %{ "eonw $dst, $src1, zr" %}
9895
9896 ins_encode %{
9897 __ eonw(as_Register($dst$$reg),
9898 as_Register($src1$$reg),
9899 zr,
9900 Assembler::LSL, 0);
9901 %}
9902
9903 ins_pipe(ialu_reg);
9904 %}
9905
9906 // This pattern is automatically generated from aarch64_ad.m4
9907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9908 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9909 immI0 zero, iRegIorL2I src1, immI src2) %{
9910 match(Set dst (SubI zero (URShiftI src1 src2)));
9911
9912 ins_cost(1.9 * INSN_COST);
9913 format %{ "negw $dst, $src1, LSR $src2" %}
9914
9915 ins_encode %{
9916 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9917 Assembler::LSR, $src2$$constant & 0x1f);
9918 %}
9919
9920 ins_pipe(ialu_reg_shift);
9921 %}
9922
9923 // This pattern is automatically generated from aarch64_ad.m4
9924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9925 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9926 immI0 zero, iRegIorL2I src1, immI src2) %{
9927 match(Set dst (SubI zero (RShiftI src1 src2)));
9928
9929 ins_cost(1.9 * INSN_COST);
9930 format %{ "negw $dst, $src1, ASR $src2" %}
9931
9932 ins_encode %{
9933 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9934 Assembler::ASR, $src2$$constant & 0x1f);
9935 %}
9936
9937 ins_pipe(ialu_reg_shift);
9938 %}
9939
9940 // This pattern is automatically generated from aarch64_ad.m4
9941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9942 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9943 immI0 zero, iRegIorL2I src1, immI src2) %{
9944 match(Set dst (SubI zero (LShiftI src1 src2)));
9945
9946 ins_cost(1.9 * INSN_COST);
9947 format %{ "negw $dst, $src1, LSL $src2" %}
9948
9949 ins_encode %{
9950 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9951 Assembler::LSL, $src2$$constant & 0x1f);
9952 %}
9953
9954 ins_pipe(ialu_reg_shift);
9955 %}
9956
9957 // This pattern is automatically generated from aarch64_ad.m4
9958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9959 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9960 immL0 zero, iRegL src1, immI src2) %{
9961 match(Set dst (SubL zero (URShiftL src1 src2)));
9962
9963 ins_cost(1.9 * INSN_COST);
9964 format %{ "neg $dst, $src1, LSR $src2" %}
9965
9966 ins_encode %{
9967 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9968 Assembler::LSR, $src2$$constant & 0x3f);
9969 %}
9970
9971 ins_pipe(ialu_reg_shift);
9972 %}
9973
9974 // This pattern is automatically generated from aarch64_ad.m4
9975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9976 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9977 immL0 zero, iRegL src1, immI src2) %{
9978 match(Set dst (SubL zero (RShiftL src1 src2)));
9979
9980 ins_cost(1.9 * INSN_COST);
9981 format %{ "neg $dst, $src1, ASR $src2" %}
9982
9983 ins_encode %{
9984 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9985 Assembler::ASR, $src2$$constant & 0x3f);
9986 %}
9987
9988 ins_pipe(ialu_reg_shift);
9989 %}
9990
9991 // This pattern is automatically generated from aarch64_ad.m4
9992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9993 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
9994 immL0 zero, iRegL src1, immI src2) %{
9995 match(Set dst (SubL zero (LShiftL src1 src2)));
9996
9997 ins_cost(1.9 * INSN_COST);
9998 format %{ "neg $dst, $src1, LSL $src2" %}
9999
10000 ins_encode %{
10001 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10002 Assembler::LSL, $src2$$constant & 0x3f);
10003 %}
10004
10005 ins_pipe(ialu_reg_shift);
10006 %}
10007
10008 // This pattern is automatically generated from aarch64_ad.m4
10009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10010 instruct AndI_reg_not_reg(iRegINoSp dst,
10011 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10012 match(Set dst (AndI src1 (XorI src2 m1)));
10013 ins_cost(INSN_COST);
10014 format %{ "bicw $dst, $src1, $src2" %}
10015
10016 ins_encode %{
10017 __ bicw(as_Register($dst$$reg),
10018 as_Register($src1$$reg),
10019 as_Register($src2$$reg),
10020 Assembler::LSL, 0);
10021 %}
10022
10023 ins_pipe(ialu_reg_reg);
10024 %}
10025
10026 // This pattern is automatically generated from aarch64_ad.m4
10027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10028 instruct AndL_reg_not_reg(iRegLNoSp dst,
10029 iRegL src1, iRegL src2, immL_M1 m1) %{
10030 match(Set dst (AndL src1 (XorL src2 m1)));
10031 ins_cost(INSN_COST);
10032 format %{ "bic $dst, $src1, $src2" %}
10033
10034 ins_encode %{
10035 __ bic(as_Register($dst$$reg),
10036 as_Register($src1$$reg),
10037 as_Register($src2$$reg),
10038 Assembler::LSL, 0);
10039 %}
10040
10041 ins_pipe(ialu_reg_reg);
10042 %}
10043
10044 // This pattern is automatically generated from aarch64_ad.m4
10045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10046 instruct OrI_reg_not_reg(iRegINoSp dst,
10047 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10048 match(Set dst (OrI src1 (XorI src2 m1)));
10049 ins_cost(INSN_COST);
10050 format %{ "ornw $dst, $src1, $src2" %}
10051
10052 ins_encode %{
10053 __ ornw(as_Register($dst$$reg),
10054 as_Register($src1$$reg),
10055 as_Register($src2$$reg),
10056 Assembler::LSL, 0);
10057 %}
10058
10059 ins_pipe(ialu_reg_reg);
10060 %}
10061
10062 // This pattern is automatically generated from aarch64_ad.m4
10063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10064 instruct OrL_reg_not_reg(iRegLNoSp dst,
10065 iRegL src1, iRegL src2, immL_M1 m1) %{
10066 match(Set dst (OrL src1 (XorL src2 m1)));
10067 ins_cost(INSN_COST);
10068 format %{ "orn $dst, $src1, $src2" %}
10069
10070 ins_encode %{
10071 __ orn(as_Register($dst$$reg),
10072 as_Register($src1$$reg),
10073 as_Register($src2$$reg),
10074 Assembler::LSL, 0);
10075 %}
10076
10077 ins_pipe(ialu_reg_reg);
10078 %}
10079
10080 // This pattern is automatically generated from aarch64_ad.m4
10081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10082 instruct XorI_reg_not_reg(iRegINoSp dst,
10083 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10084 match(Set dst (XorI m1 (XorI src2 src1)));
10085 ins_cost(INSN_COST);
10086 format %{ "eonw $dst, $src1, $src2" %}
10087
10088 ins_encode %{
10089 __ eonw(as_Register($dst$$reg),
10090 as_Register($src1$$reg),
10091 as_Register($src2$$reg),
10092 Assembler::LSL, 0);
10093 %}
10094
10095 ins_pipe(ialu_reg_reg);
10096 %}
10097
10098 // This pattern is automatically generated from aarch64_ad.m4
10099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10100 instruct XorL_reg_not_reg(iRegLNoSp dst,
10101 iRegL src1, iRegL src2, immL_M1 m1) %{
10102 match(Set dst (XorL m1 (XorL src2 src1)));
10103 ins_cost(INSN_COST);
10104 format %{ "eon $dst, $src1, $src2" %}
10105
10106 ins_encode %{
10107 __ eon(as_Register($dst$$reg),
10108 as_Register($src1$$reg),
10109 as_Register($src2$$reg),
10110 Assembler::LSL, 0);
10111 %}
10112
10113 ins_pipe(ialu_reg_reg);
10114 %}
10115
10116 // This pattern is automatically generated from aarch64_ad.m4
10117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10118 // val & (-1 ^ (val >>> shift)) ==> bicw
10119 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10120 iRegIorL2I src1, iRegIorL2I src2,
10121 immI src3, immI_M1 src4) %{
10122 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10123 ins_cost(1.9 * INSN_COST);
10124 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10125
10126 ins_encode %{
10127 __ bicw(as_Register($dst$$reg),
10128 as_Register($src1$$reg),
10129 as_Register($src2$$reg),
10130 Assembler::LSR,
10131 $src3$$constant & 0x1f);
10132 %}
10133
10134 ins_pipe(ialu_reg_reg_shift);
10135 %}
10136
10137 // This pattern is automatically generated from aarch64_ad.m4
10138 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10139 // val & (-1 ^ (val >>> shift)) ==> bic
10140 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10141 iRegL src1, iRegL src2,
10142 immI src3, immL_M1 src4) %{
10143 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10144 ins_cost(1.9 * INSN_COST);
10145 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10146
10147 ins_encode %{
10148 __ bic(as_Register($dst$$reg),
10149 as_Register($src1$$reg),
10150 as_Register($src2$$reg),
10151 Assembler::LSR,
10152 $src3$$constant & 0x3f);
10153 %}
10154
10155 ins_pipe(ialu_reg_reg_shift);
10156 %}
10157
10158 // This pattern is automatically generated from aarch64_ad.m4
10159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10160 // val & (-1 ^ (val >> shift)) ==> bicw
10161 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10162 iRegIorL2I src1, iRegIorL2I src2,
10163 immI src3, immI_M1 src4) %{
10164 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10165 ins_cost(1.9 * INSN_COST);
10166 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10167
10168 ins_encode %{
10169 __ bicw(as_Register($dst$$reg),
10170 as_Register($src1$$reg),
10171 as_Register($src2$$reg),
10172 Assembler::ASR,
10173 $src3$$constant & 0x1f);
10174 %}
10175
10176 ins_pipe(ialu_reg_reg_shift);
10177 %}
10178
10179 // This pattern is automatically generated from aarch64_ad.m4
10180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10181 // val & (-1 ^ (val >> shift)) ==> bic
10182 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10183 iRegL src1, iRegL src2,
10184 immI src3, immL_M1 src4) %{
10185 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10186 ins_cost(1.9 * INSN_COST);
10187 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10188
10189 ins_encode %{
10190 __ bic(as_Register($dst$$reg),
10191 as_Register($src1$$reg),
10192 as_Register($src2$$reg),
10193 Assembler::ASR,
10194 $src3$$constant & 0x3f);
10195 %}
10196
10197 ins_pipe(ialu_reg_reg_shift);
10198 %}
10199
10200 // This pattern is automatically generated from aarch64_ad.m4
10201 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10202 // val & (-1 ^ (val ror shift)) ==> bicw
10203 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10204 iRegIorL2I src1, iRegIorL2I src2,
10205 immI src3, immI_M1 src4) %{
10206 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10207 ins_cost(1.9 * INSN_COST);
10208 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10209
10210 ins_encode %{
10211 __ bicw(as_Register($dst$$reg),
10212 as_Register($src1$$reg),
10213 as_Register($src2$$reg),
10214 Assembler::ROR,
10215 $src3$$constant & 0x1f);
10216 %}
10217
10218 ins_pipe(ialu_reg_reg_shift);
10219 %}
10220
10221 // This pattern is automatically generated from aarch64_ad.m4
10222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10223 // val & (-1 ^ (val ror shift)) ==> bic
10224 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10225 iRegL src1, iRegL src2,
10226 immI src3, immL_M1 src4) %{
10227 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10228 ins_cost(1.9 * INSN_COST);
10229 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10230
10231 ins_encode %{
10232 __ bic(as_Register($dst$$reg),
10233 as_Register($src1$$reg),
10234 as_Register($src2$$reg),
10235 Assembler::ROR,
10236 $src3$$constant & 0x3f);
10237 %}
10238
10239 ins_pipe(ialu_reg_reg_shift);
10240 %}
10241
10242 // This pattern is automatically generated from aarch64_ad.m4
10243 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10244 // val & (-1 ^ (val << shift)) ==> bicw
10245 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10246 iRegIorL2I src1, iRegIorL2I src2,
10247 immI src3, immI_M1 src4) %{
10248 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10249 ins_cost(1.9 * INSN_COST);
10250 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10251
10252 ins_encode %{
10253 __ bicw(as_Register($dst$$reg),
10254 as_Register($src1$$reg),
10255 as_Register($src2$$reg),
10256 Assembler::LSL,
10257 $src3$$constant & 0x1f);
10258 %}
10259
10260 ins_pipe(ialu_reg_reg_shift);
10261 %}
10262
10263 // This pattern is automatically generated from aarch64_ad.m4
10264 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10265 // val & (-1 ^ (val << shift)) ==> bic
10266 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10267 iRegL src1, iRegL src2,
10268 immI src3, immL_M1 src4) %{
10269 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10270 ins_cost(1.9 * INSN_COST);
10271 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10272
10273 ins_encode %{
10274 __ bic(as_Register($dst$$reg),
10275 as_Register($src1$$reg),
10276 as_Register($src2$$reg),
10277 Assembler::LSL,
10278 $src3$$constant & 0x3f);
10279 %}
10280
10281 ins_pipe(ialu_reg_reg_shift);
10282 %}
10283
10284 // This pattern is automatically generated from aarch64_ad.m4
10285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10286 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10287 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10288 iRegIorL2I src1, iRegIorL2I src2,
10289 immI src3, immI_M1 src4) %{
10290 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10291 ins_cost(1.9 * INSN_COST);
10292 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10293
10294 ins_encode %{
10295 __ eonw(as_Register($dst$$reg),
10296 as_Register($src1$$reg),
10297 as_Register($src2$$reg),
10298 Assembler::LSR,
10299 $src3$$constant & 0x1f);
10300 %}
10301
10302 ins_pipe(ialu_reg_reg_shift);
10303 %}
10304
10305 // This pattern is automatically generated from aarch64_ad.m4
10306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10307 // val ^ (-1 ^ (val >>> shift)) ==> eon
10308 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10309 iRegL src1, iRegL src2,
10310 immI src3, immL_M1 src4) %{
10311 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10312 ins_cost(1.9 * INSN_COST);
10313 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10314
10315 ins_encode %{
10316 __ eon(as_Register($dst$$reg),
10317 as_Register($src1$$reg),
10318 as_Register($src2$$reg),
10319 Assembler::LSR,
10320 $src3$$constant & 0x3f);
10321 %}
10322
10323 ins_pipe(ialu_reg_reg_shift);
10324 %}
10325
10326 // This pattern is automatically generated from aarch64_ad.m4
10327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10328 // val ^ (-1 ^ (val >> shift)) ==> eonw
10329 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10330 iRegIorL2I src1, iRegIorL2I src2,
10331 immI src3, immI_M1 src4) %{
10332 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10333 ins_cost(1.9 * INSN_COST);
10334 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10335
10336 ins_encode %{
10337 __ eonw(as_Register($dst$$reg),
10338 as_Register($src1$$reg),
10339 as_Register($src2$$reg),
10340 Assembler::ASR,
10341 $src3$$constant & 0x1f);
10342 %}
10343
10344 ins_pipe(ialu_reg_reg_shift);
10345 %}
10346
10347 // This pattern is automatically generated from aarch64_ad.m4
10348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10349 // val ^ (-1 ^ (val >> shift)) ==> eon
10350 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10351 iRegL src1, iRegL src2,
10352 immI src3, immL_M1 src4) %{
10353 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10354 ins_cost(1.9 * INSN_COST);
10355 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10356
10357 ins_encode %{
10358 __ eon(as_Register($dst$$reg),
10359 as_Register($src1$$reg),
10360 as_Register($src2$$reg),
10361 Assembler::ASR,
10362 $src3$$constant & 0x3f);
10363 %}
10364
10365 ins_pipe(ialu_reg_reg_shift);
10366 %}
10367
10368 // This pattern is automatically generated from aarch64_ad.m4
10369 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10370 // val ^ (-1 ^ (val ror shift)) ==> eonw
10371 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10372 iRegIorL2I src1, iRegIorL2I src2,
10373 immI src3, immI_M1 src4) %{
10374 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10375 ins_cost(1.9 * INSN_COST);
10376 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10377
10378 ins_encode %{
10379 __ eonw(as_Register($dst$$reg),
10380 as_Register($src1$$reg),
10381 as_Register($src2$$reg),
10382 Assembler::ROR,
10383 $src3$$constant & 0x1f);
10384 %}
10385
10386 ins_pipe(ialu_reg_reg_shift);
10387 %}
10388
10389 // This pattern is automatically generated from aarch64_ad.m4
10390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10391 // val ^ (-1 ^ (val ror shift)) ==> eon
10392 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10393 iRegL src1, iRegL src2,
10394 immI src3, immL_M1 src4) %{
10395 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10396 ins_cost(1.9 * INSN_COST);
10397 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10398
10399 ins_encode %{
10400 __ eon(as_Register($dst$$reg),
10401 as_Register($src1$$reg),
10402 as_Register($src2$$reg),
10403 Assembler::ROR,
10404 $src3$$constant & 0x3f);
10405 %}
10406
10407 ins_pipe(ialu_reg_reg_shift);
10408 %}
10409
10410 // This pattern is automatically generated from aarch64_ad.m4
10411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10412 // val ^ (-1 ^ (val << shift)) ==> eonw
10413 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10414 iRegIorL2I src1, iRegIorL2I src2,
10415 immI src3, immI_M1 src4) %{
10416 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10417 ins_cost(1.9 * INSN_COST);
10418 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10419
10420 ins_encode %{
10421 __ eonw(as_Register($dst$$reg),
10422 as_Register($src1$$reg),
10423 as_Register($src2$$reg),
10424 Assembler::LSL,
10425 $src3$$constant & 0x1f);
10426 %}
10427
10428 ins_pipe(ialu_reg_reg_shift);
10429 %}
10430
10431 // This pattern is automatically generated from aarch64_ad.m4
10432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10433 // val ^ (-1 ^ (val << shift)) ==> eon
10434 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10435 iRegL src1, iRegL src2,
10436 immI src3, immL_M1 src4) %{
10437 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10438 ins_cost(1.9 * INSN_COST);
10439 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10440
10441 ins_encode %{
10442 __ eon(as_Register($dst$$reg),
10443 as_Register($src1$$reg),
10444 as_Register($src2$$reg),
10445 Assembler::LSL,
10446 $src3$$constant & 0x3f);
10447 %}
10448
10449 ins_pipe(ialu_reg_reg_shift);
10450 %}
10451
10452 // This pattern is automatically generated from aarch64_ad.m4
10453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10454 // val | (-1 ^ (val >>> shift)) ==> ornw
10455 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10456 iRegIorL2I src1, iRegIorL2I src2,
10457 immI src3, immI_M1 src4) %{
10458 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10459 ins_cost(1.9 * INSN_COST);
10460 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10461
10462 ins_encode %{
10463 __ ornw(as_Register($dst$$reg),
10464 as_Register($src1$$reg),
10465 as_Register($src2$$reg),
10466 Assembler::LSR,
10467 $src3$$constant & 0x1f);
10468 %}
10469
10470 ins_pipe(ialu_reg_reg_shift);
10471 %}
10472
10473 // This pattern is automatically generated from aarch64_ad.m4
10474 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10475 // val | (-1 ^ (val >>> shift)) ==> orn
10476 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10477 iRegL src1, iRegL src2,
10478 immI src3, immL_M1 src4) %{
10479 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10480 ins_cost(1.9 * INSN_COST);
10481 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10482
10483 ins_encode %{
10484 __ orn(as_Register($dst$$reg),
10485 as_Register($src1$$reg),
10486 as_Register($src2$$reg),
10487 Assembler::LSR,
10488 $src3$$constant & 0x3f);
10489 %}
10490
10491 ins_pipe(ialu_reg_reg_shift);
10492 %}
10493
10494 // This pattern is automatically generated from aarch64_ad.m4
10495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10496 // val | (-1 ^ (val >> shift)) ==> ornw
10497 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10498 iRegIorL2I src1, iRegIorL2I src2,
10499 immI src3, immI_M1 src4) %{
10500 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10501 ins_cost(1.9 * INSN_COST);
10502 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10503
10504 ins_encode %{
10505 __ ornw(as_Register($dst$$reg),
10506 as_Register($src1$$reg),
10507 as_Register($src2$$reg),
10508 Assembler::ASR,
10509 $src3$$constant & 0x1f);
10510 %}
10511
10512 ins_pipe(ialu_reg_reg_shift);
10513 %}
10514
10515 // This pattern is automatically generated from aarch64_ad.m4
10516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10517 // val | (-1 ^ (val >> shift)) ==> orn
10518 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10519 iRegL src1, iRegL src2,
10520 immI src3, immL_M1 src4) %{
10521 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10522 ins_cost(1.9 * INSN_COST);
10523 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10524
10525 ins_encode %{
10526 __ orn(as_Register($dst$$reg),
10527 as_Register($src1$$reg),
10528 as_Register($src2$$reg),
10529 Assembler::ASR,
10530 $src3$$constant & 0x3f);
10531 %}
10532
10533 ins_pipe(ialu_reg_reg_shift);
10534 %}
10535
10536 // This pattern is automatically generated from aarch64_ad.m4
10537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10538 // val | (-1 ^ (val ror shift)) ==> ornw
10539 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10540 iRegIorL2I src1, iRegIorL2I src2,
10541 immI src3, immI_M1 src4) %{
10542 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10543 ins_cost(1.9 * INSN_COST);
10544 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10545
10546 ins_encode %{
10547 __ ornw(as_Register($dst$$reg),
10548 as_Register($src1$$reg),
10549 as_Register($src2$$reg),
10550 Assembler::ROR,
10551 $src3$$constant & 0x1f);
10552 %}
10553
10554 ins_pipe(ialu_reg_reg_shift);
10555 %}
10556
10557 // This pattern is automatically generated from aarch64_ad.m4
10558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10559 // val | (-1 ^ (val ror shift)) ==> orn
10560 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10561 iRegL src1, iRegL src2,
10562 immI src3, immL_M1 src4) %{
10563 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10564 ins_cost(1.9 * INSN_COST);
10565 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10566
10567 ins_encode %{
10568 __ orn(as_Register($dst$$reg),
10569 as_Register($src1$$reg),
10570 as_Register($src2$$reg),
10571 Assembler::ROR,
10572 $src3$$constant & 0x3f);
10573 %}
10574
10575 ins_pipe(ialu_reg_reg_shift);
10576 %}
10577
10578 // This pattern is automatically generated from aarch64_ad.m4
10579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10580 // val | (-1 ^ (val << shift)) ==> ornw
10581 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10582 iRegIorL2I src1, iRegIorL2I src2,
10583 immI src3, immI_M1 src4) %{
10584 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10585 ins_cost(1.9 * INSN_COST);
10586 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10587
10588 ins_encode %{
10589 __ ornw(as_Register($dst$$reg),
10590 as_Register($src1$$reg),
10591 as_Register($src2$$reg),
10592 Assembler::LSL,
10593 $src3$$constant & 0x1f);
10594 %}
10595
10596 ins_pipe(ialu_reg_reg_shift);
10597 %}
10598
10599 // This pattern is automatically generated from aarch64_ad.m4
10600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10601 // val | (-1 ^ (val << shift)) ==> orn
10602 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10603 iRegL src1, iRegL src2,
10604 immI src3, immL_M1 src4) %{
10605 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10606 ins_cost(1.9 * INSN_COST);
10607 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10608
10609 ins_encode %{
10610 __ orn(as_Register($dst$$reg),
10611 as_Register($src1$$reg),
10612 as_Register($src2$$reg),
10613 Assembler::LSL,
10614 $src3$$constant & 0x3f);
10615 %}
10616
10617 ins_pipe(ialu_reg_reg_shift);
10618 %}
10619
10620 // This pattern is automatically generated from aarch64_ad.m4
10621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10622 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10623 iRegIorL2I src1, iRegIorL2I src2,
10624 immI src3) %{
10625 match(Set dst (AndI src1 (URShiftI src2 src3)));
10626
10627 ins_cost(1.9 * INSN_COST);
10628 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10629
10630 ins_encode %{
10631 __ andw(as_Register($dst$$reg),
10632 as_Register($src1$$reg),
10633 as_Register($src2$$reg),
10634 Assembler::LSR,
10635 $src3$$constant & 0x1f);
10636 %}
10637
10638 ins_pipe(ialu_reg_reg_shift);
10639 %}
10640
10641 // This pattern is automatically generated from aarch64_ad.m4
10642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10643 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10644 iRegL src1, iRegL src2,
10645 immI src3) %{
10646 match(Set dst (AndL src1 (URShiftL src2 src3)));
10647
10648 ins_cost(1.9 * INSN_COST);
10649 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10650
10651 ins_encode %{
10652 __ andr(as_Register($dst$$reg),
10653 as_Register($src1$$reg),
10654 as_Register($src2$$reg),
10655 Assembler::LSR,
10656 $src3$$constant & 0x3f);
10657 %}
10658
10659 ins_pipe(ialu_reg_reg_shift);
10660 %}
10661
10662 // This pattern is automatically generated from aarch64_ad.m4
10663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10664 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10665 iRegIorL2I src1, iRegIorL2I src2,
10666 immI src3) %{
10667 match(Set dst (AndI src1 (RShiftI src2 src3)));
10668
10669 ins_cost(1.9 * INSN_COST);
10670 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10671
10672 ins_encode %{
10673 __ andw(as_Register($dst$$reg),
10674 as_Register($src1$$reg),
10675 as_Register($src2$$reg),
10676 Assembler::ASR,
10677 $src3$$constant & 0x1f);
10678 %}
10679
10680 ins_pipe(ialu_reg_reg_shift);
10681 %}
10682
10683 // This pattern is automatically generated from aarch64_ad.m4
10684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10685 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10686 iRegL src1, iRegL src2,
10687 immI src3) %{
10688 match(Set dst (AndL src1 (RShiftL src2 src3)));
10689
10690 ins_cost(1.9 * INSN_COST);
10691 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10692
10693 ins_encode %{
10694 __ andr(as_Register($dst$$reg),
10695 as_Register($src1$$reg),
10696 as_Register($src2$$reg),
10697 Assembler::ASR,
10698 $src3$$constant & 0x3f);
10699 %}
10700
10701 ins_pipe(ialu_reg_reg_shift);
10702 %}
10703
10704 // This pattern is automatically generated from aarch64_ad.m4
10705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10706 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10707 iRegIorL2I src1, iRegIorL2I src2,
10708 immI src3) %{
10709 match(Set dst (AndI src1 (LShiftI src2 src3)));
10710
10711 ins_cost(1.9 * INSN_COST);
10712 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10713
10714 ins_encode %{
10715 __ andw(as_Register($dst$$reg),
10716 as_Register($src1$$reg),
10717 as_Register($src2$$reg),
10718 Assembler::LSL,
10719 $src3$$constant & 0x1f);
10720 %}
10721
10722 ins_pipe(ialu_reg_reg_shift);
10723 %}
10724
10725 // This pattern is automatically generated from aarch64_ad.m4
10726 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10727 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10728 iRegL src1, iRegL src2,
10729 immI src3) %{
10730 match(Set dst (AndL src1 (LShiftL src2 src3)));
10731
10732 ins_cost(1.9 * INSN_COST);
10733 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10734
10735 ins_encode %{
10736 __ andr(as_Register($dst$$reg),
10737 as_Register($src1$$reg),
10738 as_Register($src2$$reg),
10739 Assembler::LSL,
10740 $src3$$constant & 0x3f);
10741 %}
10742
10743 ins_pipe(ialu_reg_reg_shift);
10744 %}
10745
10746 // This pattern is automatically generated from aarch64_ad.m4
10747 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10748 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10749 iRegIorL2I src1, iRegIorL2I src2,
10750 immI src3) %{
10751 match(Set dst (AndI src1 (RotateRight src2 src3)));
10752
10753 ins_cost(1.9 * INSN_COST);
10754 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10755
10756 ins_encode %{
10757 __ andw(as_Register($dst$$reg),
10758 as_Register($src1$$reg),
10759 as_Register($src2$$reg),
10760 Assembler::ROR,
10761 $src3$$constant & 0x1f);
10762 %}
10763
10764 ins_pipe(ialu_reg_reg_shift);
10765 %}
10766
10767 // This pattern is automatically generated from aarch64_ad.m4
10768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10769 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10770 iRegL src1, iRegL src2,
10771 immI src3) %{
10772 match(Set dst (AndL src1 (RotateRight src2 src3)));
10773
10774 ins_cost(1.9 * INSN_COST);
10775 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10776
10777 ins_encode %{
10778 __ andr(as_Register($dst$$reg),
10779 as_Register($src1$$reg),
10780 as_Register($src2$$reg),
10781 Assembler::ROR,
10782 $src3$$constant & 0x3f);
10783 %}
10784
10785 ins_pipe(ialu_reg_reg_shift);
10786 %}
10787
10788 // This pattern is automatically generated from aarch64_ad.m4
10789 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10790 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10791 iRegIorL2I src1, iRegIorL2I src2,
10792 immI src3) %{
10793 match(Set dst (XorI src1 (URShiftI src2 src3)));
10794
10795 ins_cost(1.9 * INSN_COST);
10796 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10797
10798 ins_encode %{
10799 __ eorw(as_Register($dst$$reg),
10800 as_Register($src1$$reg),
10801 as_Register($src2$$reg),
10802 Assembler::LSR,
10803 $src3$$constant & 0x1f);
10804 %}
10805
10806 ins_pipe(ialu_reg_reg_shift);
10807 %}
10808
10809 // This pattern is automatically generated from aarch64_ad.m4
10810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10811 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10812 iRegL src1, iRegL src2,
10813 immI src3) %{
10814 match(Set dst (XorL src1 (URShiftL src2 src3)));
10815
10816 ins_cost(1.9 * INSN_COST);
10817 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10818
10819 ins_encode %{
10820 __ eor(as_Register($dst$$reg),
10821 as_Register($src1$$reg),
10822 as_Register($src2$$reg),
10823 Assembler::LSR,
10824 $src3$$constant & 0x3f);
10825 %}
10826
10827 ins_pipe(ialu_reg_reg_shift);
10828 %}
10829
10830 // This pattern is automatically generated from aarch64_ad.m4
10831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10832 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10833 iRegIorL2I src1, iRegIorL2I src2,
10834 immI src3) %{
10835 match(Set dst (XorI src1 (RShiftI src2 src3)));
10836
10837 ins_cost(1.9 * INSN_COST);
10838 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10839
10840 ins_encode %{
10841 __ eorw(as_Register($dst$$reg),
10842 as_Register($src1$$reg),
10843 as_Register($src2$$reg),
10844 Assembler::ASR,
10845 $src3$$constant & 0x1f);
10846 %}
10847
10848 ins_pipe(ialu_reg_reg_shift);
10849 %}
10850
10851 // This pattern is automatically generated from aarch64_ad.m4
10852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10853 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10854 iRegL src1, iRegL src2,
10855 immI src3) %{
10856 match(Set dst (XorL src1 (RShiftL src2 src3)));
10857
10858 ins_cost(1.9 * INSN_COST);
10859 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10860
10861 ins_encode %{
10862 __ eor(as_Register($dst$$reg),
10863 as_Register($src1$$reg),
10864 as_Register($src2$$reg),
10865 Assembler::ASR,
10866 $src3$$constant & 0x3f);
10867 %}
10868
10869 ins_pipe(ialu_reg_reg_shift);
10870 %}
10871
10872 // This pattern is automatically generated from aarch64_ad.m4
10873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10874 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10875 iRegIorL2I src1, iRegIorL2I src2,
10876 immI src3) %{
10877 match(Set dst (XorI src1 (LShiftI src2 src3)));
10878
10879 ins_cost(1.9 * INSN_COST);
10880 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10881
10882 ins_encode %{
10883 __ eorw(as_Register($dst$$reg),
10884 as_Register($src1$$reg),
10885 as_Register($src2$$reg),
10886 Assembler::LSL,
10887 $src3$$constant & 0x1f);
10888 %}
10889
10890 ins_pipe(ialu_reg_reg_shift);
10891 %}
10892
10893 // This pattern is automatically generated from aarch64_ad.m4
10894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10895 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10896 iRegL src1, iRegL src2,
10897 immI src3) %{
10898 match(Set dst (XorL src1 (LShiftL src2 src3)));
10899
10900 ins_cost(1.9 * INSN_COST);
10901 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10902
10903 ins_encode %{
10904 __ eor(as_Register($dst$$reg),
10905 as_Register($src1$$reg),
10906 as_Register($src2$$reg),
10907 Assembler::LSL,
10908 $src3$$constant & 0x3f);
10909 %}
10910
10911 ins_pipe(ialu_reg_reg_shift);
10912 %}
10913
10914 // This pattern is automatically generated from aarch64_ad.m4
10915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10916 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10917 iRegIorL2I src1, iRegIorL2I src2,
10918 immI src3) %{
10919 match(Set dst (XorI src1 (RotateRight src2 src3)));
10920
10921 ins_cost(1.9 * INSN_COST);
10922 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10923
10924 ins_encode %{
10925 __ eorw(as_Register($dst$$reg),
10926 as_Register($src1$$reg),
10927 as_Register($src2$$reg),
10928 Assembler::ROR,
10929 $src3$$constant & 0x1f);
10930 %}
10931
10932 ins_pipe(ialu_reg_reg_shift);
10933 %}
10934
10935 // This pattern is automatically generated from aarch64_ad.m4
10936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10937 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10938 iRegL src1, iRegL src2,
10939 immI src3) %{
10940 match(Set dst (XorL src1 (RotateRight src2 src3)));
10941
10942 ins_cost(1.9 * INSN_COST);
10943 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10944
10945 ins_encode %{
10946 __ eor(as_Register($dst$$reg),
10947 as_Register($src1$$reg),
10948 as_Register($src2$$reg),
10949 Assembler::ROR,
10950 $src3$$constant & 0x3f);
10951 %}
10952
10953 ins_pipe(ialu_reg_reg_shift);
10954 %}
10955
10956 // This pattern is automatically generated from aarch64_ad.m4
10957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10958 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10959 iRegIorL2I src1, iRegIorL2I src2,
10960 immI src3) %{
10961 match(Set dst (OrI src1 (URShiftI src2 src3)));
10962
10963 ins_cost(1.9 * INSN_COST);
10964 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10965
10966 ins_encode %{
10967 __ orrw(as_Register($dst$$reg),
10968 as_Register($src1$$reg),
10969 as_Register($src2$$reg),
10970 Assembler::LSR,
10971 $src3$$constant & 0x1f);
10972 %}
10973
10974 ins_pipe(ialu_reg_reg_shift);
10975 %}
10976
10977 // This pattern is automatically generated from aarch64_ad.m4
10978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10979 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10980 iRegL src1, iRegL src2,
10981 immI src3) %{
10982 match(Set dst (OrL src1 (URShiftL src2 src3)));
10983
10984 ins_cost(1.9 * INSN_COST);
10985 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10986
10987 ins_encode %{
10988 __ orr(as_Register($dst$$reg),
10989 as_Register($src1$$reg),
10990 as_Register($src2$$reg),
10991 Assembler::LSR,
10992 $src3$$constant & 0x3f);
10993 %}
10994
10995 ins_pipe(ialu_reg_reg_shift);
10996 %}
10997
10998 // This pattern is automatically generated from aarch64_ad.m4
10999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11000 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11001 iRegIorL2I src1, iRegIorL2I src2,
11002 immI src3) %{
11003 match(Set dst (OrI src1 (RShiftI src2 src3)));
11004
11005 ins_cost(1.9 * INSN_COST);
11006 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11007
11008 ins_encode %{
11009 __ orrw(as_Register($dst$$reg),
11010 as_Register($src1$$reg),
11011 as_Register($src2$$reg),
11012 Assembler::ASR,
11013 $src3$$constant & 0x1f);
11014 %}
11015
11016 ins_pipe(ialu_reg_reg_shift);
11017 %}
11018
11019 // This pattern is automatically generated from aarch64_ad.m4
11020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11021 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11022 iRegL src1, iRegL src2,
11023 immI src3) %{
11024 match(Set dst (OrL src1 (RShiftL src2 src3)));
11025
11026 ins_cost(1.9 * INSN_COST);
11027 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11028
11029 ins_encode %{
11030 __ orr(as_Register($dst$$reg),
11031 as_Register($src1$$reg),
11032 as_Register($src2$$reg),
11033 Assembler::ASR,
11034 $src3$$constant & 0x3f);
11035 %}
11036
11037 ins_pipe(ialu_reg_reg_shift);
11038 %}
11039
11040 // This pattern is automatically generated from aarch64_ad.m4
11041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11042 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11043 iRegIorL2I src1, iRegIorL2I src2,
11044 immI src3) %{
11045 match(Set dst (OrI src1 (LShiftI src2 src3)));
11046
11047 ins_cost(1.9 * INSN_COST);
11048 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11049
11050 ins_encode %{
11051 __ orrw(as_Register($dst$$reg),
11052 as_Register($src1$$reg),
11053 as_Register($src2$$reg),
11054 Assembler::LSL,
11055 $src3$$constant & 0x1f);
11056 %}
11057
11058 ins_pipe(ialu_reg_reg_shift);
11059 %}
11060
11061 // This pattern is automatically generated from aarch64_ad.m4
11062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11063 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11064 iRegL src1, iRegL src2,
11065 immI src3) %{
11066 match(Set dst (OrL src1 (LShiftL src2 src3)));
11067
11068 ins_cost(1.9 * INSN_COST);
11069 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11070
11071 ins_encode %{
11072 __ orr(as_Register($dst$$reg),
11073 as_Register($src1$$reg),
11074 as_Register($src2$$reg),
11075 Assembler::LSL,
11076 $src3$$constant & 0x3f);
11077 %}
11078
11079 ins_pipe(ialu_reg_reg_shift);
11080 %}
11081
11082 // This pattern is automatically generated from aarch64_ad.m4
11083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11084 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11085 iRegIorL2I src1, iRegIorL2I src2,
11086 immI src3) %{
11087 match(Set dst (OrI src1 (RotateRight src2 src3)));
11088
11089 ins_cost(1.9 * INSN_COST);
11090 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11091
11092 ins_encode %{
11093 __ orrw(as_Register($dst$$reg),
11094 as_Register($src1$$reg),
11095 as_Register($src2$$reg),
11096 Assembler::ROR,
11097 $src3$$constant & 0x1f);
11098 %}
11099
11100 ins_pipe(ialu_reg_reg_shift);
11101 %}
11102
11103 // This pattern is automatically generated from aarch64_ad.m4
11104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11105 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11106 iRegL src1, iRegL src2,
11107 immI src3) %{
11108 match(Set dst (OrL src1 (RotateRight src2 src3)));
11109
11110 ins_cost(1.9 * INSN_COST);
11111 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11112
11113 ins_encode %{
11114 __ orr(as_Register($dst$$reg),
11115 as_Register($src1$$reg),
11116 as_Register($src2$$reg),
11117 Assembler::ROR,
11118 $src3$$constant & 0x3f);
11119 %}
11120
11121 ins_pipe(ialu_reg_reg_shift);
11122 %}
11123
11124 // This pattern is automatically generated from aarch64_ad.m4
11125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11126 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11127 iRegIorL2I src1, iRegIorL2I src2,
11128 immI src3) %{
11129 match(Set dst (AddI src1 (URShiftI src2 src3)));
11130
11131 ins_cost(1.9 * INSN_COST);
11132 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11133
11134 ins_encode %{
11135 __ addw(as_Register($dst$$reg),
11136 as_Register($src1$$reg),
11137 as_Register($src2$$reg),
11138 Assembler::LSR,
11139 $src3$$constant & 0x1f);
11140 %}
11141
11142 ins_pipe(ialu_reg_reg_shift);
11143 %}
11144
11145 // This pattern is automatically generated from aarch64_ad.m4
11146 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11147 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11148 iRegL src1, iRegL src2,
11149 immI src3) %{
11150 match(Set dst (AddL src1 (URShiftL src2 src3)));
11151
11152 ins_cost(1.9 * INSN_COST);
11153 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11154
11155 ins_encode %{
11156 __ add(as_Register($dst$$reg),
11157 as_Register($src1$$reg),
11158 as_Register($src2$$reg),
11159 Assembler::LSR,
11160 $src3$$constant & 0x3f);
11161 %}
11162
11163 ins_pipe(ialu_reg_reg_shift);
11164 %}
11165
11166 // This pattern is automatically generated from aarch64_ad.m4
11167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11168 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11169 iRegIorL2I src1, iRegIorL2I src2,
11170 immI src3) %{
11171 match(Set dst (AddI src1 (RShiftI src2 src3)));
11172
11173 ins_cost(1.9 * INSN_COST);
11174 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11175
11176 ins_encode %{
11177 __ addw(as_Register($dst$$reg),
11178 as_Register($src1$$reg),
11179 as_Register($src2$$reg),
11180 Assembler::ASR,
11181 $src3$$constant & 0x1f);
11182 %}
11183
11184 ins_pipe(ialu_reg_reg_shift);
11185 %}
11186
11187 // This pattern is automatically generated from aarch64_ad.m4
11188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11189 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11190 iRegL src1, iRegL src2,
11191 immI src3) %{
11192 match(Set dst (AddL src1 (RShiftL src2 src3)));
11193
11194 ins_cost(1.9 * INSN_COST);
11195 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11196
11197 ins_encode %{
11198 __ add(as_Register($dst$$reg),
11199 as_Register($src1$$reg),
11200 as_Register($src2$$reg),
11201 Assembler::ASR,
11202 $src3$$constant & 0x3f);
11203 %}
11204
11205 ins_pipe(ialu_reg_reg_shift);
11206 %}
11207
11208 // This pattern is automatically generated from aarch64_ad.m4
11209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11210 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11211 iRegIorL2I src1, iRegIorL2I src2,
11212 immI src3) %{
11213 match(Set dst (AddI src1 (LShiftI src2 src3)));
11214
11215 ins_cost(1.9 * INSN_COST);
11216 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11217
11218 ins_encode %{
11219 __ addw(as_Register($dst$$reg),
11220 as_Register($src1$$reg),
11221 as_Register($src2$$reg),
11222 Assembler::LSL,
11223 $src3$$constant & 0x1f);
11224 %}
11225
11226 ins_pipe(ialu_reg_reg_shift);
11227 %}
11228
11229 // This pattern is automatically generated from aarch64_ad.m4
11230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11231 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11232 iRegL src1, iRegL src2,
11233 immI src3) %{
11234 match(Set dst (AddL src1 (LShiftL src2 src3)));
11235
11236 ins_cost(1.9 * INSN_COST);
11237 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11238
11239 ins_encode %{
11240 __ add(as_Register($dst$$reg),
11241 as_Register($src1$$reg),
11242 as_Register($src2$$reg),
11243 Assembler::LSL,
11244 $src3$$constant & 0x3f);
11245 %}
11246
11247 ins_pipe(ialu_reg_reg_shift);
11248 %}
11249
11250 // This pattern is automatically generated from aarch64_ad.m4
11251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11252 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11253 iRegIorL2I src1, iRegIorL2I src2,
11254 immI src3) %{
11255 match(Set dst (SubI src1 (URShiftI src2 src3)));
11256
11257 ins_cost(1.9 * INSN_COST);
11258 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11259
11260 ins_encode %{
11261 __ subw(as_Register($dst$$reg),
11262 as_Register($src1$$reg),
11263 as_Register($src2$$reg),
11264 Assembler::LSR,
11265 $src3$$constant & 0x1f);
11266 %}
11267
11268 ins_pipe(ialu_reg_reg_shift);
11269 %}
11270
11271 // This pattern is automatically generated from aarch64_ad.m4
11272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11273 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11274 iRegL src1, iRegL src2,
11275 immI src3) %{
11276 match(Set dst (SubL src1 (URShiftL src2 src3)));
11277
11278 ins_cost(1.9 * INSN_COST);
11279 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11280
11281 ins_encode %{
11282 __ sub(as_Register($dst$$reg),
11283 as_Register($src1$$reg),
11284 as_Register($src2$$reg),
11285 Assembler::LSR,
11286 $src3$$constant & 0x3f);
11287 %}
11288
11289 ins_pipe(ialu_reg_reg_shift);
11290 %}
11291
11292 // This pattern is automatically generated from aarch64_ad.m4
11293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11294 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11295 iRegIorL2I src1, iRegIorL2I src2,
11296 immI src3) %{
11297 match(Set dst (SubI src1 (RShiftI src2 src3)));
11298
11299 ins_cost(1.9 * INSN_COST);
11300 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11301
11302 ins_encode %{
11303 __ subw(as_Register($dst$$reg),
11304 as_Register($src1$$reg),
11305 as_Register($src2$$reg),
11306 Assembler::ASR,
11307 $src3$$constant & 0x1f);
11308 %}
11309
11310 ins_pipe(ialu_reg_reg_shift);
11311 %}
11312
11313 // This pattern is automatically generated from aarch64_ad.m4
11314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11315 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11316 iRegL src1, iRegL src2,
11317 immI src3) %{
11318 match(Set dst (SubL src1 (RShiftL src2 src3)));
11319
11320 ins_cost(1.9 * INSN_COST);
11321 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11322
11323 ins_encode %{
11324 __ sub(as_Register($dst$$reg),
11325 as_Register($src1$$reg),
11326 as_Register($src2$$reg),
11327 Assembler::ASR,
11328 $src3$$constant & 0x3f);
11329 %}
11330
11331 ins_pipe(ialu_reg_reg_shift);
11332 %}
11333
11334 // This pattern is automatically generated from aarch64_ad.m4
11335 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11336 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11337 iRegIorL2I src1, iRegIorL2I src2,
11338 immI src3) %{
11339 match(Set dst (SubI src1 (LShiftI src2 src3)));
11340
11341 ins_cost(1.9 * INSN_COST);
11342 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11343
11344 ins_encode %{
11345 __ subw(as_Register($dst$$reg),
11346 as_Register($src1$$reg),
11347 as_Register($src2$$reg),
11348 Assembler::LSL,
11349 $src3$$constant & 0x1f);
11350 %}
11351
11352 ins_pipe(ialu_reg_reg_shift);
11353 %}
11354
11355 // This pattern is automatically generated from aarch64_ad.m4
11356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11357 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11358 iRegL src1, iRegL src2,
11359 immI src3) %{
11360 match(Set dst (SubL src1 (LShiftL src2 src3)));
11361
11362 ins_cost(1.9 * INSN_COST);
11363 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11364
11365 ins_encode %{
11366 __ sub(as_Register($dst$$reg),
11367 as_Register($src1$$reg),
11368 as_Register($src2$$reg),
11369 Assembler::LSL,
11370 $src3$$constant & 0x3f);
11371 %}
11372
11373 ins_pipe(ialu_reg_reg_shift);
11374 %}
11375
11376 // This pattern is automatically generated from aarch64_ad.m4
11377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11378
11379 // Shift Left followed by Shift Right.
11380 // This idiom is used by the compiler for the i2b bytecode etc.
11381 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11382 %{
11383 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11384 ins_cost(INSN_COST * 2);
11385 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11386 ins_encode %{
11387 int lshift = $lshift_count$$constant & 63;
11388 int rshift = $rshift_count$$constant & 63;
11389 int s = 63 - lshift;
11390 int r = (rshift - lshift) & 63;
11391 __ sbfm(as_Register($dst$$reg),
11392 as_Register($src$$reg),
11393 r, s);
11394 %}
11395
11396 ins_pipe(ialu_reg_shift);
11397 %}
11398
11399 // This pattern is automatically generated from aarch64_ad.m4
11400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11401
11402 // Shift Left followed by Shift Right.
11403 // This idiom is used by the compiler for the i2b bytecode etc.
11404 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11405 %{
11406 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11407 ins_cost(INSN_COST * 2);
11408 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11409 ins_encode %{
11410 int lshift = $lshift_count$$constant & 31;
11411 int rshift = $rshift_count$$constant & 31;
11412 int s = 31 - lshift;
11413 int r = (rshift - lshift) & 31;
11414 __ sbfmw(as_Register($dst$$reg),
11415 as_Register($src$$reg),
11416 r, s);
11417 %}
11418
11419 ins_pipe(ialu_reg_shift);
11420 %}
11421
11422 // This pattern is automatically generated from aarch64_ad.m4
11423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11424
11425 // Shift Left followed by Shift Right.
11426 // This idiom is used by the compiler for the i2b bytecode etc.
11427 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11428 %{
11429 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11430 ins_cost(INSN_COST * 2);
11431 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11432 ins_encode %{
11433 int lshift = $lshift_count$$constant & 63;
11434 int rshift = $rshift_count$$constant & 63;
11435 int s = 63 - lshift;
11436 int r = (rshift - lshift) & 63;
11437 __ ubfm(as_Register($dst$$reg),
11438 as_Register($src$$reg),
11439 r, s);
11440 %}
11441
11442 ins_pipe(ialu_reg_shift);
11443 %}
11444
11445 // This pattern is automatically generated from aarch64_ad.m4
11446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11447
11448 // Shift Left followed by Shift Right.
11449 // This idiom is used by the compiler for the i2b bytecode etc.
11450 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11451 %{
11452 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11453 ins_cost(INSN_COST * 2);
11454 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11455 ins_encode %{
11456 int lshift = $lshift_count$$constant & 31;
11457 int rshift = $rshift_count$$constant & 31;
11458 int s = 31 - lshift;
11459 int r = (rshift - lshift) & 31;
11460 __ ubfmw(as_Register($dst$$reg),
11461 as_Register($src$$reg),
11462 r, s);
11463 %}
11464
11465 ins_pipe(ialu_reg_shift);
11466 %}
11467
11468 // Bitfield extract with shift & mask
11469
11470 // This pattern is automatically generated from aarch64_ad.m4
11471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11472 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11473 %{
11474 match(Set dst (AndI (URShiftI src rshift) mask));
11475 // Make sure we are not going to exceed what ubfxw can do.
11476 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11477
11478 ins_cost(INSN_COST);
11479 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11480 ins_encode %{
11481 int rshift = $rshift$$constant & 31;
11482 intptr_t mask = $mask$$constant;
11483 int width = exact_log2(mask+1);
11484 __ ubfxw(as_Register($dst$$reg),
11485 as_Register($src$$reg), rshift, width);
11486 %}
11487 ins_pipe(ialu_reg_shift);
11488 %}
11489
11490 // This pattern is automatically generated from aarch64_ad.m4
11491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11492 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11493 %{
11494 match(Set dst (AndL (URShiftL src rshift) mask));
11495 // Make sure we are not going to exceed what ubfx can do.
11496 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11497
11498 ins_cost(INSN_COST);
11499 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11500 ins_encode %{
11501 int rshift = $rshift$$constant & 63;
11502 intptr_t mask = $mask$$constant;
11503 int width = exact_log2_long(mask+1);
11504 __ ubfx(as_Register($dst$$reg),
11505 as_Register($src$$reg), rshift, width);
11506 %}
11507 ins_pipe(ialu_reg_shift);
11508 %}
11509
11510
11511 // This pattern is automatically generated from aarch64_ad.m4
11512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11513
11514 // We can use ubfx when extending an And with a mask when we know mask
11515 // is positive. We know that because immI_bitmask guarantees it.
11516 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11517 %{
11518 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11519 // Make sure we are not going to exceed what ubfxw can do.
11520 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11521
11522 ins_cost(INSN_COST * 2);
11523 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11524 ins_encode %{
11525 int rshift = $rshift$$constant & 31;
11526 intptr_t mask = $mask$$constant;
11527 int width = exact_log2(mask+1);
11528 __ ubfx(as_Register($dst$$reg),
11529 as_Register($src$$reg), rshift, width);
11530 %}
11531 ins_pipe(ialu_reg_shift);
11532 %}
11533
11534
11535 // This pattern is automatically generated from aarch64_ad.m4
11536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11537
11538 // We can use ubfiz when masking by a positive number and then left shifting the result.
11539 // We know that the mask is positive because immI_bitmask guarantees it.
11540 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11541 %{
11542 match(Set dst (LShiftI (AndI src mask) lshift));
11543 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11544
11545 ins_cost(INSN_COST);
11546 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11547 ins_encode %{
11548 int lshift = $lshift$$constant & 31;
11549 intptr_t mask = $mask$$constant;
11550 int width = exact_log2(mask+1);
11551 __ ubfizw(as_Register($dst$$reg),
11552 as_Register($src$$reg), lshift, width);
11553 %}
11554 ins_pipe(ialu_reg_shift);
11555 %}
11556
11557 // This pattern is automatically generated from aarch64_ad.m4
11558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11559
11560 // We can use ubfiz when masking by a positive number and then left shifting the result.
11561 // We know that the mask is positive because immL_bitmask guarantees it.
11562 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11563 %{
11564 match(Set dst (LShiftL (AndL src mask) lshift));
11565 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11566
11567 ins_cost(INSN_COST);
11568 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11569 ins_encode %{
11570 int lshift = $lshift$$constant & 63;
11571 intptr_t mask = $mask$$constant;
11572 int width = exact_log2_long(mask+1);
11573 __ ubfiz(as_Register($dst$$reg),
11574 as_Register($src$$reg), lshift, width);
11575 %}
11576 ins_pipe(ialu_reg_shift);
11577 %}
11578
11579 // This pattern is automatically generated from aarch64_ad.m4
11580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11581
11582 // We can use ubfiz when masking by a positive number and then left shifting the result.
11583 // We know that the mask is positive because immI_bitmask guarantees it.
11584 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11585 %{
11586 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11587 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11588
11589 ins_cost(INSN_COST);
11590 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11591 ins_encode %{
11592 int lshift = $lshift$$constant & 31;
11593 intptr_t mask = $mask$$constant;
11594 int width = exact_log2(mask+1);
11595 __ ubfizw(as_Register($dst$$reg),
11596 as_Register($src$$reg), lshift, width);
11597 %}
11598 ins_pipe(ialu_reg_shift);
11599 %}
11600
11601 // This pattern is automatically generated from aarch64_ad.m4
11602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11603
11604 // We can use ubfiz when masking by a positive number and then left shifting the result.
11605 // We know that the mask is positive because immL_bitmask guarantees it.
11606 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11607 %{
11608 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11609 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11610
11611 ins_cost(INSN_COST);
11612 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11613 ins_encode %{
11614 int lshift = $lshift$$constant & 63;
11615 intptr_t mask = $mask$$constant;
11616 int width = exact_log2_long(mask+1);
11617 __ ubfiz(as_Register($dst$$reg),
11618 as_Register($src$$reg), lshift, width);
11619 %}
11620 ins_pipe(ialu_reg_shift);
11621 %}
11622
11623
11624 // This pattern is automatically generated from aarch64_ad.m4
11625 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11626
11627 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11628 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11629 %{
11630 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11631 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11632
11633 ins_cost(INSN_COST);
11634 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11635 ins_encode %{
11636 int lshift = $lshift$$constant & 63;
11637 intptr_t mask = $mask$$constant;
11638 int width = exact_log2(mask+1);
11639 __ ubfiz(as_Register($dst$$reg),
11640 as_Register($src$$reg), lshift, width);
11641 %}
11642 ins_pipe(ialu_reg_shift);
11643 %}
11644
11645 // This pattern is automatically generated from aarch64_ad.m4
11646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11647
11648 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11649 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11650 %{
11651 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11652 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11653
11654 ins_cost(INSN_COST);
11655 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11656 ins_encode %{
11657 int lshift = $lshift$$constant & 31;
11658 intptr_t mask = $mask$$constant;
11659 int width = exact_log2(mask+1);
11660 __ ubfiz(as_Register($dst$$reg),
11661 as_Register($src$$reg), lshift, width);
11662 %}
11663 ins_pipe(ialu_reg_shift);
11664 %}
11665
11666 // This pattern is automatically generated from aarch64_ad.m4
11667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11668
11669 // Can skip int2long conversions after AND with small bitmask
11670 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11671 %{
11672 match(Set dst (ConvI2L (AndI src msk)));
11673 ins_cost(INSN_COST);
11674 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11675 ins_encode %{
11676 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11677 %}
11678 ins_pipe(ialu_reg_shift);
11679 %}
11680
11681
11682 // Rotations
11683
11684 // This pattern is automatically generated from aarch64_ad.m4
11685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11686 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11687 %{
11688 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11689 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11690
11691 ins_cost(INSN_COST);
11692 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11693
11694 ins_encode %{
11695 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11696 $rshift$$constant & 63);
11697 %}
11698 ins_pipe(ialu_reg_reg_extr);
11699 %}
11700
11701
11702 // This pattern is automatically generated from aarch64_ad.m4
11703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11704 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11705 %{
11706 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11707 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11708
11709 ins_cost(INSN_COST);
11710 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11711
11712 ins_encode %{
11713 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11714 $rshift$$constant & 31);
11715 %}
11716 ins_pipe(ialu_reg_reg_extr);
11717 %}
11718
11719
11720 // This pattern is automatically generated from aarch64_ad.m4
11721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11722 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11723 %{
11724 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11725 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11726
11727 ins_cost(INSN_COST);
11728 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11729
11730 ins_encode %{
11731 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11732 $rshift$$constant & 63);
11733 %}
11734 ins_pipe(ialu_reg_reg_extr);
11735 %}
11736
11737
11738 // This pattern is automatically generated from aarch64_ad.m4
11739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11740 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11741 %{
11742 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11743 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11744
11745 ins_cost(INSN_COST);
11746 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11747
11748 ins_encode %{
11749 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11750 $rshift$$constant & 31);
11751 %}
11752 ins_pipe(ialu_reg_reg_extr);
11753 %}
11754
11755 // This pattern is automatically generated from aarch64_ad.m4
11756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11757 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11758 %{
11759 match(Set dst (RotateRight src shift));
11760
11761 ins_cost(INSN_COST);
11762 format %{ "ror $dst, $src, $shift" %}
11763
11764 ins_encode %{
11765 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11766 $shift$$constant & 0x1f);
11767 %}
11768 ins_pipe(ialu_reg_reg_vshift);
11769 %}
11770
11771 // This pattern is automatically generated from aarch64_ad.m4
11772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11773 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11774 %{
11775 match(Set dst (RotateRight src shift));
11776
11777 ins_cost(INSN_COST);
11778 format %{ "ror $dst, $src, $shift" %}
11779
11780 ins_encode %{
11781 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11782 $shift$$constant & 0x3f);
11783 %}
11784 ins_pipe(ialu_reg_reg_vshift);
11785 %}
11786
11787 // This pattern is automatically generated from aarch64_ad.m4
11788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11789 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11790 %{
11791 match(Set dst (RotateRight src shift));
11792
11793 ins_cost(INSN_COST);
11794 format %{ "ror $dst, $src, $shift" %}
11795
11796 ins_encode %{
11797 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11798 %}
11799 ins_pipe(ialu_reg_reg_vshift);
11800 %}
11801
11802 // This pattern is automatically generated from aarch64_ad.m4
11803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11804 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11805 %{
11806 match(Set dst (RotateRight src shift));
11807
11808 ins_cost(INSN_COST);
11809 format %{ "ror $dst, $src, $shift" %}
11810
11811 ins_encode %{
11812 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11813 %}
11814 ins_pipe(ialu_reg_reg_vshift);
11815 %}
11816
11817 // This pattern is automatically generated from aarch64_ad.m4
11818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11819 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11820 %{
11821 match(Set dst (RotateLeft src shift));
11822
11823 ins_cost(INSN_COST);
11824 format %{ "rol $dst, $src, $shift" %}
11825
11826 ins_encode %{
11827 __ subw(rscratch1, zr, as_Register($shift$$reg));
11828 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11829 %}
11830 ins_pipe(ialu_reg_reg_vshift);
11831 %}
11832
11833 // This pattern is automatically generated from aarch64_ad.m4
11834 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11835 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11836 %{
11837 match(Set dst (RotateLeft src shift));
11838
11839 ins_cost(INSN_COST);
11840 format %{ "rol $dst, $src, $shift" %}
11841
11842 ins_encode %{
11843 __ subw(rscratch1, zr, as_Register($shift$$reg));
11844 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11845 %}
11846 ins_pipe(ialu_reg_reg_vshift);
11847 %}
11848
11849
11850 // Add/subtract (extended)
11851
11852 // This pattern is automatically generated from aarch64_ad.m4
11853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11854 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11855 %{
11856 match(Set dst (AddL src1 (ConvI2L src2)));
11857 ins_cost(INSN_COST);
11858 format %{ "add $dst, $src1, $src2, sxtw" %}
11859
11860 ins_encode %{
11861 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11862 as_Register($src2$$reg), ext::sxtw);
11863 %}
11864 ins_pipe(ialu_reg_reg);
11865 %}
11866
11867 // This pattern is automatically generated from aarch64_ad.m4
11868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11869 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11870 %{
11871 match(Set dst (SubL src1 (ConvI2L src2)));
11872 ins_cost(INSN_COST);
11873 format %{ "sub $dst, $src1, $src2, sxtw" %}
11874
11875 ins_encode %{
11876 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11877 as_Register($src2$$reg), ext::sxtw);
11878 %}
11879 ins_pipe(ialu_reg_reg);
11880 %}
11881
11882 // This pattern is automatically generated from aarch64_ad.m4
11883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11884 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11885 %{
11886 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11887 ins_cost(INSN_COST);
11888 format %{ "add $dst, $src1, $src2, sxth" %}
11889
11890 ins_encode %{
11891 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11892 as_Register($src2$$reg), ext::sxth);
11893 %}
11894 ins_pipe(ialu_reg_reg);
11895 %}
11896
11897 // This pattern is automatically generated from aarch64_ad.m4
11898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11899 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11900 %{
11901 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11902 ins_cost(INSN_COST);
11903 format %{ "add $dst, $src1, $src2, sxtb" %}
11904
11905 ins_encode %{
11906 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11907 as_Register($src2$$reg), ext::sxtb);
11908 %}
11909 ins_pipe(ialu_reg_reg);
11910 %}
11911
11912 // This pattern is automatically generated from aarch64_ad.m4
11913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11914 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11915 %{
11916 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11917 ins_cost(INSN_COST);
11918 format %{ "add $dst, $src1, $src2, uxtb" %}
11919
11920 ins_encode %{
11921 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11922 as_Register($src2$$reg), ext::uxtb);
11923 %}
11924 ins_pipe(ialu_reg_reg);
11925 %}
11926
11927 // This pattern is automatically generated from aarch64_ad.m4
11928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11929 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11930 %{
11931 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11932 ins_cost(INSN_COST);
11933 format %{ "add $dst, $src1, $src2, sxth" %}
11934
11935 ins_encode %{
11936 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11937 as_Register($src2$$reg), ext::sxth);
11938 %}
11939 ins_pipe(ialu_reg_reg);
11940 %}
11941
11942 // This pattern is automatically generated from aarch64_ad.m4
11943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11944 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11945 %{
11946 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11947 ins_cost(INSN_COST);
11948 format %{ "add $dst, $src1, $src2, sxtw" %}
11949
11950 ins_encode %{
11951 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11952 as_Register($src2$$reg), ext::sxtw);
11953 %}
11954 ins_pipe(ialu_reg_reg);
11955 %}
11956
11957 // This pattern is automatically generated from aarch64_ad.m4
11958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11959 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11960 %{
11961 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11962 ins_cost(INSN_COST);
11963 format %{ "add $dst, $src1, $src2, sxtb" %}
11964
11965 ins_encode %{
11966 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11967 as_Register($src2$$reg), ext::sxtb);
11968 %}
11969 ins_pipe(ialu_reg_reg);
11970 %}
11971
11972 // This pattern is automatically generated from aarch64_ad.m4
11973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11974 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11975 %{
11976 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11977 ins_cost(INSN_COST);
11978 format %{ "add $dst, $src1, $src2, uxtb" %}
11979
11980 ins_encode %{
11981 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11982 as_Register($src2$$reg), ext::uxtb);
11983 %}
11984 ins_pipe(ialu_reg_reg);
11985 %}
11986
11987 // This pattern is automatically generated from aarch64_ad.m4
11988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11989 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11990 %{
11991 match(Set dst (AddI src1 (AndI src2 mask)));
11992 ins_cost(INSN_COST);
11993 format %{ "addw $dst, $src1, $src2, uxtb" %}
11994
11995 ins_encode %{
11996 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11997 as_Register($src2$$reg), ext::uxtb);
11998 %}
11999 ins_pipe(ialu_reg_reg);
12000 %}
12001
12002 // This pattern is automatically generated from aarch64_ad.m4
12003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12004 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12005 %{
12006 match(Set dst (AddI src1 (AndI src2 mask)));
12007 ins_cost(INSN_COST);
12008 format %{ "addw $dst, $src1, $src2, uxth" %}
12009
12010 ins_encode %{
12011 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12012 as_Register($src2$$reg), ext::uxth);
12013 %}
12014 ins_pipe(ialu_reg_reg);
12015 %}
12016
12017 // This pattern is automatically generated from aarch64_ad.m4
12018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12019 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12020 %{
12021 match(Set dst (AddL src1 (AndL src2 mask)));
12022 ins_cost(INSN_COST);
12023 format %{ "add $dst, $src1, $src2, uxtb" %}
12024
12025 ins_encode %{
12026 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12027 as_Register($src2$$reg), ext::uxtb);
12028 %}
12029 ins_pipe(ialu_reg_reg);
12030 %}
12031
12032 // This pattern is automatically generated from aarch64_ad.m4
12033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12034 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12035 %{
12036 match(Set dst (AddL src1 (AndL src2 mask)));
12037 ins_cost(INSN_COST);
12038 format %{ "add $dst, $src1, $src2, uxth" %}
12039
12040 ins_encode %{
12041 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12042 as_Register($src2$$reg), ext::uxth);
12043 %}
12044 ins_pipe(ialu_reg_reg);
12045 %}
12046
12047 // This pattern is automatically generated from aarch64_ad.m4
12048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12049 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12050 %{
12051 match(Set dst (AddL src1 (AndL src2 mask)));
12052 ins_cost(INSN_COST);
12053 format %{ "add $dst, $src1, $src2, uxtw" %}
12054
12055 ins_encode %{
12056 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12057 as_Register($src2$$reg), ext::uxtw);
12058 %}
12059 ins_pipe(ialu_reg_reg);
12060 %}
12061
12062 // This pattern is automatically generated from aarch64_ad.m4
12063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12064 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12065 %{
12066 match(Set dst (SubI src1 (AndI src2 mask)));
12067 ins_cost(INSN_COST);
12068 format %{ "subw $dst, $src1, $src2, uxtb" %}
12069
12070 ins_encode %{
12071 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12072 as_Register($src2$$reg), ext::uxtb);
12073 %}
12074 ins_pipe(ialu_reg_reg);
12075 %}
12076
12077 // This pattern is automatically generated from aarch64_ad.m4
12078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12079 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12080 %{
12081 match(Set dst (SubI src1 (AndI src2 mask)));
12082 ins_cost(INSN_COST);
12083 format %{ "subw $dst, $src1, $src2, uxth" %}
12084
12085 ins_encode %{
12086 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12087 as_Register($src2$$reg), ext::uxth);
12088 %}
12089 ins_pipe(ialu_reg_reg);
12090 %}
12091
12092 // This pattern is automatically generated from aarch64_ad.m4
12093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12094 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12095 %{
12096 match(Set dst (SubL src1 (AndL src2 mask)));
12097 ins_cost(INSN_COST);
12098 format %{ "sub $dst, $src1, $src2, uxtb" %}
12099
12100 ins_encode %{
12101 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12102 as_Register($src2$$reg), ext::uxtb);
12103 %}
12104 ins_pipe(ialu_reg_reg);
12105 %}
12106
12107 // This pattern is automatically generated from aarch64_ad.m4
12108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12109 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12110 %{
12111 match(Set dst (SubL src1 (AndL src2 mask)));
12112 ins_cost(INSN_COST);
12113 format %{ "sub $dst, $src1, $src2, uxth" %}
12114
12115 ins_encode %{
12116 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12117 as_Register($src2$$reg), ext::uxth);
12118 %}
12119 ins_pipe(ialu_reg_reg);
12120 %}
12121
12122 // This pattern is automatically generated from aarch64_ad.m4
12123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12124 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12125 %{
12126 match(Set dst (SubL src1 (AndL src2 mask)));
12127 ins_cost(INSN_COST);
12128 format %{ "sub $dst, $src1, $src2, uxtw" %}
12129
12130 ins_encode %{
12131 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12132 as_Register($src2$$reg), ext::uxtw);
12133 %}
12134 ins_pipe(ialu_reg_reg);
12135 %}
12136
12137
12138 // This pattern is automatically generated from aarch64_ad.m4
12139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12140 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12141 %{
12142 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12143 ins_cost(1.9 * INSN_COST);
12144 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12145
12146 ins_encode %{
12147 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12148 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12149 %}
12150 ins_pipe(ialu_reg_reg_shift);
12151 %}
12152
12153 // This pattern is automatically generated from aarch64_ad.m4
12154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12155 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12156 %{
12157 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12158 ins_cost(1.9 * INSN_COST);
12159 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12160
12161 ins_encode %{
12162 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12163 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12164 %}
12165 ins_pipe(ialu_reg_reg_shift);
12166 %}
12167
12168 // This pattern is automatically generated from aarch64_ad.m4
12169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12170 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12171 %{
12172 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12173 ins_cost(1.9 * INSN_COST);
12174 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12175
12176 ins_encode %{
12177 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12178 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12179 %}
12180 ins_pipe(ialu_reg_reg_shift);
12181 %}
12182
12183 // This pattern is automatically generated from aarch64_ad.m4
12184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12185 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12186 %{
12187 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12188 ins_cost(1.9 * INSN_COST);
12189 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12190
12191 ins_encode %{
12192 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12193 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12194 %}
12195 ins_pipe(ialu_reg_reg_shift);
12196 %}
12197
12198 // This pattern is automatically generated from aarch64_ad.m4
12199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12200 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12201 %{
12202 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12203 ins_cost(1.9 * INSN_COST);
12204 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12205
12206 ins_encode %{
12207 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12208 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12209 %}
12210 ins_pipe(ialu_reg_reg_shift);
12211 %}
12212
12213 // This pattern is automatically generated from aarch64_ad.m4
12214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12215 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12216 %{
12217 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12218 ins_cost(1.9 * INSN_COST);
12219 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12220
12221 ins_encode %{
12222 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12223 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12224 %}
12225 ins_pipe(ialu_reg_reg_shift);
12226 %}
12227
12228 // This pattern is automatically generated from aarch64_ad.m4
12229 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12230 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12231 %{
12232 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12233 ins_cost(1.9 * INSN_COST);
12234 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12235
12236 ins_encode %{
12237 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12238 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12239 %}
12240 ins_pipe(ialu_reg_reg_shift);
12241 %}
12242
12243 // This pattern is automatically generated from aarch64_ad.m4
12244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12245 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12246 %{
12247 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12248 ins_cost(1.9 * INSN_COST);
12249 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12250
12251 ins_encode %{
12252 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12253 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12254 %}
12255 ins_pipe(ialu_reg_reg_shift);
12256 %}
12257
12258 // This pattern is automatically generated from aarch64_ad.m4
12259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12260 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12261 %{
12262 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12263 ins_cost(1.9 * INSN_COST);
12264 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12265
12266 ins_encode %{
12267 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12268 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12269 %}
12270 ins_pipe(ialu_reg_reg_shift);
12271 %}
12272
12273 // This pattern is automatically generated from aarch64_ad.m4
12274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12275 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12276 %{
12277 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12278 ins_cost(1.9 * INSN_COST);
12279 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12280
12281 ins_encode %{
12282 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12283 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12284 %}
12285 ins_pipe(ialu_reg_reg_shift);
12286 %}
12287
12288 // This pattern is automatically generated from aarch64_ad.m4
12289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12290 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12291 %{
12292 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12293 ins_cost(1.9 * INSN_COST);
12294 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12295
12296 ins_encode %{
12297 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12298 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12299 %}
12300 ins_pipe(ialu_reg_reg_shift);
12301 %}
12302
12303 // This pattern is automatically generated from aarch64_ad.m4
12304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12305 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12306 %{
12307 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12308 ins_cost(1.9 * INSN_COST);
12309 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12310
12311 ins_encode %{
12312 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12313 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12314 %}
12315 ins_pipe(ialu_reg_reg_shift);
12316 %}
12317
12318 // This pattern is automatically generated from aarch64_ad.m4
12319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12320 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12321 %{
12322 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12323 ins_cost(1.9 * INSN_COST);
12324 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12325
12326 ins_encode %{
12327 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12328 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12329 %}
12330 ins_pipe(ialu_reg_reg_shift);
12331 %}
12332
12333 // This pattern is automatically generated from aarch64_ad.m4
12334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12335 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12336 %{
12337 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12338 ins_cost(1.9 * INSN_COST);
12339 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12340
12341 ins_encode %{
12342 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12343 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12344 %}
12345 ins_pipe(ialu_reg_reg_shift);
12346 %}
12347
12348 // This pattern is automatically generated from aarch64_ad.m4
12349 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12350 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12351 %{
12352 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12353 ins_cost(1.9 * INSN_COST);
12354 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12355
12356 ins_encode %{
12357 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12358 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12359 %}
12360 ins_pipe(ialu_reg_reg_shift);
12361 %}
12362
12363 // This pattern is automatically generated from aarch64_ad.m4
12364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12365 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12366 %{
12367 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12368 ins_cost(1.9 * INSN_COST);
12369 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12370
12371 ins_encode %{
12372 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12373 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12374 %}
12375 ins_pipe(ialu_reg_reg_shift);
12376 %}
12377
12378 // This pattern is automatically generated from aarch64_ad.m4
12379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12380 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12381 %{
12382 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12383 ins_cost(1.9 * INSN_COST);
12384 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12385
12386 ins_encode %{
12387 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12388 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12389 %}
12390 ins_pipe(ialu_reg_reg_shift);
12391 %}
12392
12393 // This pattern is automatically generated from aarch64_ad.m4
12394 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12395 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12396 %{
12397 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12398 ins_cost(1.9 * INSN_COST);
12399 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12400
12401 ins_encode %{
12402 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12403 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12404 %}
12405 ins_pipe(ialu_reg_reg_shift);
12406 %}
12407
12408 // This pattern is automatically generated from aarch64_ad.m4
12409 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12410 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12411 %{
12412 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12413 ins_cost(1.9 * INSN_COST);
12414 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12415
12416 ins_encode %{
12417 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12418 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12419 %}
12420 ins_pipe(ialu_reg_reg_shift);
12421 %}
12422
12423 // This pattern is automatically generated from aarch64_ad.m4
12424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12425 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12426 %{
12427 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12428 ins_cost(1.9 * INSN_COST);
12429 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12430
12431 ins_encode %{
12432 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12433 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12434 %}
12435 ins_pipe(ialu_reg_reg_shift);
12436 %}
12437
12438 // This pattern is automatically generated from aarch64_ad.m4
12439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12440 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12441 %{
12442 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12443 ins_cost(1.9 * INSN_COST);
12444 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12445
12446 ins_encode %{
12447 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12448 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12449 %}
12450 ins_pipe(ialu_reg_reg_shift);
12451 %}
12452
12453 // This pattern is automatically generated from aarch64_ad.m4
12454 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12455 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12456 %{
12457 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12458 ins_cost(1.9 * INSN_COST);
12459 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12460
12461 ins_encode %{
12462 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12463 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12464 %}
12465 ins_pipe(ialu_reg_reg_shift);
12466 %}
12467
12468 // This pattern is automatically generated from aarch64_ad.m4
12469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12470 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12471 %{
12472 effect(DEF dst, USE src1, USE src2, USE cr);
12473 ins_cost(INSN_COST * 2);
12474 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12475
12476 ins_encode %{
12477 __ cselw($dst$$Register,
12478 $src1$$Register,
12479 $src2$$Register,
12480 Assembler::LT);
12481 %}
12482 ins_pipe(icond_reg_reg);
12483 %}
12484
12485 // This pattern is automatically generated from aarch64_ad.m4
12486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12487 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12488 %{
12489 effect(DEF dst, USE src1, USE src2, USE cr);
12490 ins_cost(INSN_COST * 2);
12491 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12492
12493 ins_encode %{
12494 __ cselw($dst$$Register,
12495 $src1$$Register,
12496 $src2$$Register,
12497 Assembler::GT);
12498 %}
12499 ins_pipe(icond_reg_reg);
12500 %}
12501
12502 // This pattern is automatically generated from aarch64_ad.m4
12503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12504 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12505 %{
12506 effect(DEF dst, USE src1, USE cr);
12507 ins_cost(INSN_COST * 2);
12508 format %{ "cselw $dst, $src1, zr lt\t" %}
12509
12510 ins_encode %{
12511 __ cselw($dst$$Register,
12512 $src1$$Register,
12513 zr,
12514 Assembler::LT);
12515 %}
12516 ins_pipe(icond_reg);
12517 %}
12518
12519 // This pattern is automatically generated from aarch64_ad.m4
12520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12521 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12522 %{
12523 effect(DEF dst, USE src1, USE cr);
12524 ins_cost(INSN_COST * 2);
12525 format %{ "cselw $dst, $src1, zr gt\t" %}
12526
12527 ins_encode %{
12528 __ cselw($dst$$Register,
12529 $src1$$Register,
12530 zr,
12531 Assembler::GT);
12532 %}
12533 ins_pipe(icond_reg);
12534 %}
12535
12536 // This pattern is automatically generated from aarch64_ad.m4
12537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12538 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12539 %{
12540 effect(DEF dst, USE src1, USE cr);
12541 ins_cost(INSN_COST * 2);
12542 format %{ "csincw $dst, $src1, zr le\t" %}
12543
12544 ins_encode %{
12545 __ csincw($dst$$Register,
12546 $src1$$Register,
12547 zr,
12548 Assembler::LE);
12549 %}
12550 ins_pipe(icond_reg);
12551 %}
12552
12553 // This pattern is automatically generated from aarch64_ad.m4
12554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12555 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12556 %{
12557 effect(DEF dst, USE src1, USE cr);
12558 ins_cost(INSN_COST * 2);
12559 format %{ "csincw $dst, $src1, zr gt\t" %}
12560
12561 ins_encode %{
12562 __ csincw($dst$$Register,
12563 $src1$$Register,
12564 zr,
12565 Assembler::GT);
12566 %}
12567 ins_pipe(icond_reg);
12568 %}
12569
12570 // This pattern is automatically generated from aarch64_ad.m4
12571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12572 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12573 %{
12574 effect(DEF dst, USE src1, USE cr);
12575 ins_cost(INSN_COST * 2);
12576 format %{ "csinvw $dst, $src1, zr lt\t" %}
12577
12578 ins_encode %{
12579 __ csinvw($dst$$Register,
12580 $src1$$Register,
12581 zr,
12582 Assembler::LT);
12583 %}
12584 ins_pipe(icond_reg);
12585 %}
12586
12587 // This pattern is automatically generated from aarch64_ad.m4
12588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12589 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12590 %{
12591 effect(DEF dst, USE src1, USE cr);
12592 ins_cost(INSN_COST * 2);
12593 format %{ "csinvw $dst, $src1, zr ge\t" %}
12594
12595 ins_encode %{
12596 __ csinvw($dst$$Register,
12597 $src1$$Register,
12598 zr,
12599 Assembler::GE);
12600 %}
12601 ins_pipe(icond_reg);
12602 %}
12603
12604 // This pattern is automatically generated from aarch64_ad.m4
12605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12606 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12607 %{
12608 match(Set dst (MinI src imm));
12609 ins_cost(INSN_COST * 3);
12610 expand %{
12611 rFlagsReg cr;
12612 compI_reg_imm0(cr, src);
12613 cmovI_reg_imm0_lt(dst, src, cr);
12614 %}
12615 %}
12616
12617 // This pattern is automatically generated from aarch64_ad.m4
12618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12619 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12620 %{
12621 match(Set dst (MinI imm src));
12622 ins_cost(INSN_COST * 3);
12623 expand %{
12624 rFlagsReg cr;
12625 compI_reg_imm0(cr, src);
12626 cmovI_reg_imm0_lt(dst, src, cr);
12627 %}
12628 %}
12629
12630 // This pattern is automatically generated from aarch64_ad.m4
12631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12632 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12633 %{
12634 match(Set dst (MinI src imm));
12635 ins_cost(INSN_COST * 3);
12636 expand %{
12637 rFlagsReg cr;
12638 compI_reg_imm0(cr, src);
12639 cmovI_reg_imm1_le(dst, src, cr);
12640 %}
12641 %}
12642
12643 // This pattern is automatically generated from aarch64_ad.m4
12644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12645 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12646 %{
12647 match(Set dst (MinI imm src));
12648 ins_cost(INSN_COST * 3);
12649 expand %{
12650 rFlagsReg cr;
12651 compI_reg_imm0(cr, src);
12652 cmovI_reg_imm1_le(dst, src, cr);
12653 %}
12654 %}
12655
12656 // This pattern is automatically generated from aarch64_ad.m4
12657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12658 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12659 %{
12660 match(Set dst (MinI src imm));
12661 ins_cost(INSN_COST * 3);
12662 expand %{
12663 rFlagsReg cr;
12664 compI_reg_imm0(cr, src);
12665 cmovI_reg_immM1_lt(dst, src, cr);
12666 %}
12667 %}
12668
12669 // This pattern is automatically generated from aarch64_ad.m4
12670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12671 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12672 %{
12673 match(Set dst (MinI imm src));
12674 ins_cost(INSN_COST * 3);
12675 expand %{
12676 rFlagsReg cr;
12677 compI_reg_imm0(cr, src);
12678 cmovI_reg_immM1_lt(dst, src, cr);
12679 %}
12680 %}
12681
12682 // This pattern is automatically generated from aarch64_ad.m4
12683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12684 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12685 %{
12686 match(Set dst (MaxI src imm));
12687 ins_cost(INSN_COST * 3);
12688 expand %{
12689 rFlagsReg cr;
12690 compI_reg_imm0(cr, src);
12691 cmovI_reg_imm0_gt(dst, src, cr);
12692 %}
12693 %}
12694
12695 // This pattern is automatically generated from aarch64_ad.m4
12696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12697 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12698 %{
12699 match(Set dst (MaxI imm src));
12700 ins_cost(INSN_COST * 3);
12701 expand %{
12702 rFlagsReg cr;
12703 compI_reg_imm0(cr, src);
12704 cmovI_reg_imm0_gt(dst, src, cr);
12705 %}
12706 %}
12707
12708 // This pattern is automatically generated from aarch64_ad.m4
12709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12710 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12711 %{
12712 match(Set dst (MaxI src imm));
12713 ins_cost(INSN_COST * 3);
12714 expand %{
12715 rFlagsReg cr;
12716 compI_reg_imm0(cr, src);
12717 cmovI_reg_imm1_gt(dst, src, cr);
12718 %}
12719 %}
12720
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12724 %{
12725 match(Set dst (MaxI imm src));
12726 ins_cost(INSN_COST * 3);
12727 expand %{
12728 rFlagsReg cr;
12729 compI_reg_imm0(cr, src);
12730 cmovI_reg_imm1_gt(dst, src, cr);
12731 %}
12732 %}
12733
12734 // This pattern is automatically generated from aarch64_ad.m4
12735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12736 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12737 %{
12738 match(Set dst (MaxI src imm));
12739 ins_cost(INSN_COST * 3);
12740 expand %{
12741 rFlagsReg cr;
12742 compI_reg_imm0(cr, src);
12743 cmovI_reg_immM1_ge(dst, src, cr);
12744 %}
12745 %}
12746
12747 // This pattern is automatically generated from aarch64_ad.m4
12748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12749 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12750 %{
12751 match(Set dst (MaxI imm src));
12752 ins_cost(INSN_COST * 3);
12753 expand %{
12754 rFlagsReg cr;
12755 compI_reg_imm0(cr, src);
12756 cmovI_reg_immM1_ge(dst, src, cr);
12757 %}
12758 %}
12759
12760 // This pattern is automatically generated from aarch64_ad.m4
12761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12762 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12763 %{
12764 match(Set dst (ReverseI src));
12765 ins_cost(INSN_COST);
12766 format %{ "rbitw $dst, $src" %}
12767 ins_encode %{
12768 __ rbitw($dst$$Register, $src$$Register);
12769 %}
12770 ins_pipe(ialu_reg);
12771 %}
12772
12773 // This pattern is automatically generated from aarch64_ad.m4
12774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12775 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12776 %{
12777 match(Set dst (ReverseL src));
12778 ins_cost(INSN_COST);
12779 format %{ "rbit $dst, $src" %}
12780 ins_encode %{
12781 __ rbit($dst$$Register, $src$$Register);
12782 %}
12783 ins_pipe(ialu_reg);
12784 %}
12785
12786
12787 // END This section of the file is automatically generated. Do not edit --------------
12788
12789
12790 // ============================================================================
12791 // Floating Point Arithmetic Instructions
12792
12793 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12794 match(Set dst (AddHF src1 src2));
12795 format %{ "faddh $dst, $src1, $src2" %}
12796 ins_encode %{
12797 __ faddh($dst$$FloatRegister,
12798 $src1$$FloatRegister,
12799 $src2$$FloatRegister);
12800 %}
12801 ins_pipe(fp_dop_reg_reg_s);
12802 %}
12803
12804 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12805 match(Set dst (AddF src1 src2));
12806
12807 ins_cost(INSN_COST * 5);
12808 format %{ "fadds $dst, $src1, $src2" %}
12809
12810 ins_encode %{
12811 __ fadds(as_FloatRegister($dst$$reg),
12812 as_FloatRegister($src1$$reg),
12813 as_FloatRegister($src2$$reg));
12814 %}
12815
12816 ins_pipe(fp_dop_reg_reg_s);
12817 %}
12818
12819 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12820 match(Set dst (AddD src1 src2));
12821
12822 ins_cost(INSN_COST * 5);
12823 format %{ "faddd $dst, $src1, $src2" %}
12824
12825 ins_encode %{
12826 __ faddd(as_FloatRegister($dst$$reg),
12827 as_FloatRegister($src1$$reg),
12828 as_FloatRegister($src2$$reg));
12829 %}
12830
12831 ins_pipe(fp_dop_reg_reg_d);
12832 %}
12833
12834 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12835 match(Set dst (SubHF src1 src2));
12836 format %{ "fsubh $dst, $src1, $src2" %}
12837 ins_encode %{
12838 __ fsubh($dst$$FloatRegister,
12839 $src1$$FloatRegister,
12840 $src2$$FloatRegister);
12841 %}
12842 ins_pipe(fp_dop_reg_reg_s);
12843 %}
12844
12845 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12846 match(Set dst (SubF src1 src2));
12847
12848 ins_cost(INSN_COST * 5);
12849 format %{ "fsubs $dst, $src1, $src2" %}
12850
12851 ins_encode %{
12852 __ fsubs(as_FloatRegister($dst$$reg),
12853 as_FloatRegister($src1$$reg),
12854 as_FloatRegister($src2$$reg));
12855 %}
12856
12857 ins_pipe(fp_dop_reg_reg_s);
12858 %}
12859
12860 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12861 match(Set dst (SubD src1 src2));
12862
12863 ins_cost(INSN_COST * 5);
12864 format %{ "fsubd $dst, $src1, $src2" %}
12865
12866 ins_encode %{
12867 __ fsubd(as_FloatRegister($dst$$reg),
12868 as_FloatRegister($src1$$reg),
12869 as_FloatRegister($src2$$reg));
12870 %}
12871
12872 ins_pipe(fp_dop_reg_reg_d);
12873 %}
12874
12875 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12876 match(Set dst (MulHF src1 src2));
12877 format %{ "fmulh $dst, $src1, $src2" %}
12878 ins_encode %{
12879 __ fmulh($dst$$FloatRegister,
12880 $src1$$FloatRegister,
12881 $src2$$FloatRegister);
12882 %}
12883 ins_pipe(fp_dop_reg_reg_s);
12884 %}
12885
12886 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12887 match(Set dst (MulF src1 src2));
12888
12889 ins_cost(INSN_COST * 6);
12890 format %{ "fmuls $dst, $src1, $src2" %}
12891
12892 ins_encode %{
12893 __ fmuls(as_FloatRegister($dst$$reg),
12894 as_FloatRegister($src1$$reg),
12895 as_FloatRegister($src2$$reg));
12896 %}
12897
12898 ins_pipe(fp_dop_reg_reg_s);
12899 %}
12900
12901 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12902 match(Set dst (MulD src1 src2));
12903
12904 ins_cost(INSN_COST * 6);
12905 format %{ "fmuld $dst, $src1, $src2" %}
12906
12907 ins_encode %{
12908 __ fmuld(as_FloatRegister($dst$$reg),
12909 as_FloatRegister($src1$$reg),
12910 as_FloatRegister($src2$$reg));
12911 %}
12912
12913 ins_pipe(fp_dop_reg_reg_d);
12914 %}
12915
12916 // src1 * src2 + src3 (half-precision float)
12917 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12918 match(Set dst (FmaHF src3 (Binary src1 src2)));
12919 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12920 ins_encode %{
12921 assert(UseFMA, "Needs FMA instructions support.");
12922 __ fmaddh($dst$$FloatRegister,
12923 $src1$$FloatRegister,
12924 $src2$$FloatRegister,
12925 $src3$$FloatRegister);
12926 %}
12927 ins_pipe(pipe_class_default);
12928 %}
12929
12930 // src1 * src2 + src3
12931 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12932 match(Set dst (FmaF src3 (Binary src1 src2)));
12933
12934 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12935
12936 ins_encode %{
12937 assert(UseFMA, "Needs FMA instructions support.");
12938 __ fmadds(as_FloatRegister($dst$$reg),
12939 as_FloatRegister($src1$$reg),
12940 as_FloatRegister($src2$$reg),
12941 as_FloatRegister($src3$$reg));
12942 %}
12943
12944 ins_pipe(pipe_class_default);
12945 %}
12946
12947 // src1 * src2 + src3
12948 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12949 match(Set dst (FmaD src3 (Binary src1 src2)));
12950
12951 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12952
12953 ins_encode %{
12954 assert(UseFMA, "Needs FMA instructions support.");
12955 __ fmaddd(as_FloatRegister($dst$$reg),
12956 as_FloatRegister($src1$$reg),
12957 as_FloatRegister($src2$$reg),
12958 as_FloatRegister($src3$$reg));
12959 %}
12960
12961 ins_pipe(pipe_class_default);
12962 %}
12963
12964 // src1 * (-src2) + src3
12965 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12966 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12967 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12968
12969 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12970
12971 ins_encode %{
12972 assert(UseFMA, "Needs FMA instructions support.");
12973 __ fmsubs(as_FloatRegister($dst$$reg),
12974 as_FloatRegister($src1$$reg),
12975 as_FloatRegister($src2$$reg),
12976 as_FloatRegister($src3$$reg));
12977 %}
12978
12979 ins_pipe(pipe_class_default);
12980 %}
12981
12982 // src1 * (-src2) + src3
12983 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12984 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12985 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12986
12987 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12988
12989 ins_encode %{
12990 assert(UseFMA, "Needs FMA instructions support.");
12991 __ fmsubd(as_FloatRegister($dst$$reg),
12992 as_FloatRegister($src1$$reg),
12993 as_FloatRegister($src2$$reg),
12994 as_FloatRegister($src3$$reg));
12995 %}
12996
12997 ins_pipe(pipe_class_default);
12998 %}
12999
13000 // src1 * (-src2) - src3
13001 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13002 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13003 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13004
13005 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13006
13007 ins_encode %{
13008 assert(UseFMA, "Needs FMA instructions support.");
13009 __ fnmadds(as_FloatRegister($dst$$reg),
13010 as_FloatRegister($src1$$reg),
13011 as_FloatRegister($src2$$reg),
13012 as_FloatRegister($src3$$reg));
13013 %}
13014
13015 ins_pipe(pipe_class_default);
13016 %}
13017
13018 // src1 * (-src2) - src3
13019 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13020 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13021 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13022
13023 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13024
13025 ins_encode %{
13026 assert(UseFMA, "Needs FMA instructions support.");
13027 __ fnmaddd(as_FloatRegister($dst$$reg),
13028 as_FloatRegister($src1$$reg),
13029 as_FloatRegister($src2$$reg),
13030 as_FloatRegister($src3$$reg));
13031 %}
13032
13033 ins_pipe(pipe_class_default);
13034 %}
13035
13036 // src1 * src2 - src3
13037 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13038 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13039
13040 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13041
13042 ins_encode %{
13043 assert(UseFMA, "Needs FMA instructions support.");
13044 __ fnmsubs(as_FloatRegister($dst$$reg),
13045 as_FloatRegister($src1$$reg),
13046 as_FloatRegister($src2$$reg),
13047 as_FloatRegister($src3$$reg));
13048 %}
13049
13050 ins_pipe(pipe_class_default);
13051 %}
13052
13053 // src1 * src2 - src3
13054 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13055 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13056
13057 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13058
13059 ins_encode %{
13060 assert(UseFMA, "Needs FMA instructions support.");
13061 // n.b. insn name should be fnmsubd
13062 __ fnmsub(as_FloatRegister($dst$$reg),
13063 as_FloatRegister($src1$$reg),
13064 as_FloatRegister($src2$$reg),
13065 as_FloatRegister($src3$$reg));
13066 %}
13067
13068 ins_pipe(pipe_class_default);
13069 %}
13070
13071 // Math.max(HH)H (half-precision float)
13072 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13073 match(Set dst (MaxHF src1 src2));
13074 format %{ "fmaxh $dst, $src1, $src2" %}
13075 ins_encode %{
13076 __ fmaxh($dst$$FloatRegister,
13077 $src1$$FloatRegister,
13078 $src2$$FloatRegister);
13079 %}
13080 ins_pipe(fp_dop_reg_reg_s);
13081 %}
13082
13083 // Math.min(HH)H (half-precision float)
13084 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13085 match(Set dst (MinHF src1 src2));
13086 format %{ "fminh $dst, $src1, $src2" %}
13087 ins_encode %{
13088 __ fminh($dst$$FloatRegister,
13089 $src1$$FloatRegister,
13090 $src2$$FloatRegister);
13091 %}
13092 ins_pipe(fp_dop_reg_reg_s);
13093 %}
13094
13095 // Math.max(FF)F
13096 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13097 match(Set dst (MaxF src1 src2));
13098
13099 format %{ "fmaxs $dst, $src1, $src2" %}
13100 ins_encode %{
13101 __ fmaxs(as_FloatRegister($dst$$reg),
13102 as_FloatRegister($src1$$reg),
13103 as_FloatRegister($src2$$reg));
13104 %}
13105
13106 ins_pipe(fp_dop_reg_reg_s);
13107 %}
13108
13109 // Math.min(FF)F
13110 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13111 match(Set dst (MinF src1 src2));
13112
13113 format %{ "fmins $dst, $src1, $src2" %}
13114 ins_encode %{
13115 __ fmins(as_FloatRegister($dst$$reg),
13116 as_FloatRegister($src1$$reg),
13117 as_FloatRegister($src2$$reg));
13118 %}
13119
13120 ins_pipe(fp_dop_reg_reg_s);
13121 %}
13122
13123 // Math.max(DD)D
13124 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13125 match(Set dst (MaxD src1 src2));
13126
13127 format %{ "fmaxd $dst, $src1, $src2" %}
13128 ins_encode %{
13129 __ fmaxd(as_FloatRegister($dst$$reg),
13130 as_FloatRegister($src1$$reg),
13131 as_FloatRegister($src2$$reg));
13132 %}
13133
13134 ins_pipe(fp_dop_reg_reg_d);
13135 %}
13136
13137 // Math.min(DD)D
13138 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13139 match(Set dst (MinD src1 src2));
13140
13141 format %{ "fmind $dst, $src1, $src2" %}
13142 ins_encode %{
13143 __ fmind(as_FloatRegister($dst$$reg),
13144 as_FloatRegister($src1$$reg),
13145 as_FloatRegister($src2$$reg));
13146 %}
13147
13148 ins_pipe(fp_dop_reg_reg_d);
13149 %}
13150
13151 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13152 match(Set dst (DivHF src1 src2));
13153 format %{ "fdivh $dst, $src1, $src2" %}
13154 ins_encode %{
13155 __ fdivh($dst$$FloatRegister,
13156 $src1$$FloatRegister,
13157 $src2$$FloatRegister);
13158 %}
13159 ins_pipe(fp_div_s);
13160 %}
13161
13162 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13163 match(Set dst (DivF src1 src2));
13164
13165 ins_cost(INSN_COST * 18);
13166 format %{ "fdivs $dst, $src1, $src2" %}
13167
13168 ins_encode %{
13169 __ fdivs(as_FloatRegister($dst$$reg),
13170 as_FloatRegister($src1$$reg),
13171 as_FloatRegister($src2$$reg));
13172 %}
13173
13174 ins_pipe(fp_div_s);
13175 %}
13176
13177 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13178 match(Set dst (DivD src1 src2));
13179
13180 ins_cost(INSN_COST * 32);
13181 format %{ "fdivd $dst, $src1, $src2" %}
13182
13183 ins_encode %{
13184 __ fdivd(as_FloatRegister($dst$$reg),
13185 as_FloatRegister($src1$$reg),
13186 as_FloatRegister($src2$$reg));
13187 %}
13188
13189 ins_pipe(fp_div_d);
13190 %}
13191
13192 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13193 match(Set dst (NegF src));
13194
13195 ins_cost(INSN_COST * 3);
13196 format %{ "fneg $dst, $src" %}
13197
13198 ins_encode %{
13199 __ fnegs(as_FloatRegister($dst$$reg),
13200 as_FloatRegister($src$$reg));
13201 %}
13202
13203 ins_pipe(fp_uop_s);
13204 %}
13205
13206 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13207 match(Set dst (NegD src));
13208
13209 ins_cost(INSN_COST * 3);
13210 format %{ "fnegd $dst, $src" %}
13211
13212 ins_encode %{
13213 __ fnegd(as_FloatRegister($dst$$reg),
13214 as_FloatRegister($src$$reg));
13215 %}
13216
13217 ins_pipe(fp_uop_d);
13218 %}
13219
13220 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13221 %{
13222 match(Set dst (AbsI src));
13223
13224 effect(KILL cr);
13225 ins_cost(INSN_COST * 2);
13226 format %{ "cmpw $src, zr\n\t"
13227 "cnegw $dst, $src, Assembler::LT\t# int abs"
13228 %}
13229
13230 ins_encode %{
13231 __ cmpw(as_Register($src$$reg), zr);
13232 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13233 %}
13234 ins_pipe(pipe_class_default);
13235 %}
13236
13237 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13238 %{
13239 match(Set dst (AbsL src));
13240
13241 effect(KILL cr);
13242 ins_cost(INSN_COST * 2);
13243 format %{ "cmp $src, zr\n\t"
13244 "cneg $dst, $src, Assembler::LT\t# long abs"
13245 %}
13246
13247 ins_encode %{
13248 __ cmp(as_Register($src$$reg), zr);
13249 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13250 %}
13251 ins_pipe(pipe_class_default);
13252 %}
13253
13254 instruct absF_reg(vRegF dst, vRegF src) %{
13255 match(Set dst (AbsF src));
13256
13257 ins_cost(INSN_COST * 3);
13258 format %{ "fabss $dst, $src" %}
13259 ins_encode %{
13260 __ fabss(as_FloatRegister($dst$$reg),
13261 as_FloatRegister($src$$reg));
13262 %}
13263
13264 ins_pipe(fp_uop_s);
13265 %}
13266
13267 instruct absD_reg(vRegD dst, vRegD src) %{
13268 match(Set dst (AbsD src));
13269
13270 ins_cost(INSN_COST * 3);
13271 format %{ "fabsd $dst, $src" %}
13272 ins_encode %{
13273 __ fabsd(as_FloatRegister($dst$$reg),
13274 as_FloatRegister($src$$reg));
13275 %}
13276
13277 ins_pipe(fp_uop_d);
13278 %}
13279
13280 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13281 match(Set dst (AbsF (SubF src1 src2)));
13282
13283 ins_cost(INSN_COST * 3);
13284 format %{ "fabds $dst, $src1, $src2" %}
13285 ins_encode %{
13286 __ fabds(as_FloatRegister($dst$$reg),
13287 as_FloatRegister($src1$$reg),
13288 as_FloatRegister($src2$$reg));
13289 %}
13290
13291 ins_pipe(fp_uop_s);
13292 %}
13293
13294 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13295 match(Set dst (AbsD (SubD src1 src2)));
13296
13297 ins_cost(INSN_COST * 3);
13298 format %{ "fabdd $dst, $src1, $src2" %}
13299 ins_encode %{
13300 __ fabdd(as_FloatRegister($dst$$reg),
13301 as_FloatRegister($src1$$reg),
13302 as_FloatRegister($src2$$reg));
13303 %}
13304
13305 ins_pipe(fp_uop_d);
13306 %}
13307
13308 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13309 match(Set dst (SqrtD src));
13310
13311 ins_cost(INSN_COST * 50);
13312 format %{ "fsqrtd $dst, $src" %}
13313 ins_encode %{
13314 __ fsqrtd(as_FloatRegister($dst$$reg),
13315 as_FloatRegister($src$$reg));
13316 %}
13317
13318 ins_pipe(fp_div_s);
13319 %}
13320
13321 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13322 match(Set dst (SqrtF src));
13323
13324 ins_cost(INSN_COST * 50);
13325 format %{ "fsqrts $dst, $src" %}
13326 ins_encode %{
13327 __ fsqrts(as_FloatRegister($dst$$reg),
13328 as_FloatRegister($src$$reg));
13329 %}
13330
13331 ins_pipe(fp_div_d);
13332 %}
13333
13334 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13335 match(Set dst (SqrtHF src));
13336 format %{ "fsqrth $dst, $src" %}
13337 ins_encode %{
13338 __ fsqrth($dst$$FloatRegister,
13339 $src$$FloatRegister);
13340 %}
13341 ins_pipe(fp_div_s);
13342 %}
13343
13344 // Math.rint, floor, ceil
13345 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13346 match(Set dst (RoundDoubleMode src rmode));
13347 format %{ "frint $dst, $src, $rmode" %}
13348 ins_encode %{
13349 switch ($rmode$$constant) {
13350 case RoundDoubleModeNode::rmode_rint:
13351 __ frintnd(as_FloatRegister($dst$$reg),
13352 as_FloatRegister($src$$reg));
13353 break;
13354 case RoundDoubleModeNode::rmode_floor:
13355 __ frintmd(as_FloatRegister($dst$$reg),
13356 as_FloatRegister($src$$reg));
13357 break;
13358 case RoundDoubleModeNode::rmode_ceil:
13359 __ frintpd(as_FloatRegister($dst$$reg),
13360 as_FloatRegister($src$$reg));
13361 break;
13362 }
13363 %}
13364 ins_pipe(fp_uop_d);
13365 %}
13366
13367 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13368 match(Set dst (CopySignD src1 (Binary src2 zero)));
13369 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13370 format %{ "CopySignD $dst $src1 $src2" %}
13371 ins_encode %{
13372 FloatRegister dst = as_FloatRegister($dst$$reg),
13373 src1 = as_FloatRegister($src1$$reg),
13374 src2 = as_FloatRegister($src2$$reg),
13375 zero = as_FloatRegister($zero$$reg);
13376 __ fnegd(dst, zero);
13377 __ bsl(dst, __ T8B, src2, src1);
13378 %}
13379 ins_pipe(fp_uop_d);
13380 %}
13381
13382 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13383 match(Set dst (CopySignF src1 src2));
13384 effect(TEMP_DEF dst, USE src1, USE src2);
13385 format %{ "CopySignF $dst $src1 $src2" %}
13386 ins_encode %{
13387 FloatRegister dst = as_FloatRegister($dst$$reg),
13388 src1 = as_FloatRegister($src1$$reg),
13389 src2 = as_FloatRegister($src2$$reg);
13390 __ movi(dst, __ T2S, 0x80, 24);
13391 __ bsl(dst, __ T8B, src2, src1);
13392 %}
13393 ins_pipe(fp_uop_d);
13394 %}
13395
13396 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13397 match(Set dst (SignumD src (Binary zero one)));
13398 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13399 format %{ "signumD $dst, $src" %}
13400 ins_encode %{
13401 FloatRegister src = as_FloatRegister($src$$reg),
13402 dst = as_FloatRegister($dst$$reg),
13403 zero = as_FloatRegister($zero$$reg),
13404 one = as_FloatRegister($one$$reg);
13405 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13406 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13407 // Bit selection instruction gets bit from "one" for each enabled bit in
13408 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13409 // NaN the whole "src" will be copied because "dst" is zero. For all other
13410 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13411 // from "src", and all other bits are copied from 1.0.
13412 __ bsl(dst, __ T8B, one, src);
13413 %}
13414 ins_pipe(fp_uop_d);
13415 %}
13416
13417 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13418 match(Set dst (SignumF src (Binary zero one)));
13419 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13420 format %{ "signumF $dst, $src" %}
13421 ins_encode %{
13422 FloatRegister src = as_FloatRegister($src$$reg),
13423 dst = as_FloatRegister($dst$$reg),
13424 zero = as_FloatRegister($zero$$reg),
13425 one = as_FloatRegister($one$$reg);
13426 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13427 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13428 // Bit selection instruction gets bit from "one" for each enabled bit in
13429 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13430 // NaN the whole "src" will be copied because "dst" is zero. For all other
13431 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13432 // from "src", and all other bits are copied from 1.0.
13433 __ bsl(dst, __ T8B, one, src);
13434 %}
13435 ins_pipe(fp_uop_d);
13436 %}
13437
13438 instruct onspinwait() %{
13439 match(OnSpinWait);
13440 ins_cost(INSN_COST);
13441
13442 format %{ "onspinwait" %}
13443
13444 ins_encode %{
13445 __ spin_wait();
13446 %}
13447 ins_pipe(pipe_class_empty);
13448 %}
13449
13450 // ============================================================================
13451 // Logical Instructions
13452
13453 // Integer Logical Instructions
13454
13455 // And Instructions
13456
13457
13458 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13459 match(Set dst (AndI src1 src2));
13460
13461 format %{ "andw $dst, $src1, $src2\t# int" %}
13462
13463 ins_cost(INSN_COST);
13464 ins_encode %{
13465 __ andw(as_Register($dst$$reg),
13466 as_Register($src1$$reg),
13467 as_Register($src2$$reg));
13468 %}
13469
13470 ins_pipe(ialu_reg_reg);
13471 %}
13472
13473 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13474 match(Set dst (AndI src1 src2));
13475
13476 format %{ "andsw $dst, $src1, $src2\t# int" %}
13477
13478 ins_cost(INSN_COST);
13479 ins_encode %{
13480 __ andw(as_Register($dst$$reg),
13481 as_Register($src1$$reg),
13482 (uint64_t)($src2$$constant));
13483 %}
13484
13485 ins_pipe(ialu_reg_imm);
13486 %}
13487
13488 // Or Instructions
13489
13490 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13491 match(Set dst (OrI src1 src2));
13492
13493 format %{ "orrw $dst, $src1, $src2\t# int" %}
13494
13495 ins_cost(INSN_COST);
13496 ins_encode %{
13497 __ orrw(as_Register($dst$$reg),
13498 as_Register($src1$$reg),
13499 as_Register($src2$$reg));
13500 %}
13501
13502 ins_pipe(ialu_reg_reg);
13503 %}
13504
13505 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13506 match(Set dst (OrI src1 src2));
13507
13508 format %{ "orrw $dst, $src1, $src2\t# int" %}
13509
13510 ins_cost(INSN_COST);
13511 ins_encode %{
13512 __ orrw(as_Register($dst$$reg),
13513 as_Register($src1$$reg),
13514 (uint64_t)($src2$$constant));
13515 %}
13516
13517 ins_pipe(ialu_reg_imm);
13518 %}
13519
13520 // Xor Instructions
13521
13522 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13523 match(Set dst (XorI src1 src2));
13524
13525 format %{ "eorw $dst, $src1, $src2\t# int" %}
13526
13527 ins_cost(INSN_COST);
13528 ins_encode %{
13529 __ eorw(as_Register($dst$$reg),
13530 as_Register($src1$$reg),
13531 as_Register($src2$$reg));
13532 %}
13533
13534 ins_pipe(ialu_reg_reg);
13535 %}
13536
13537 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13538 match(Set dst (XorI src1 src2));
13539
13540 format %{ "eorw $dst, $src1, $src2\t# int" %}
13541
13542 ins_cost(INSN_COST);
13543 ins_encode %{
13544 __ eorw(as_Register($dst$$reg),
13545 as_Register($src1$$reg),
13546 (uint64_t)($src2$$constant));
13547 %}
13548
13549 ins_pipe(ialu_reg_imm);
13550 %}
13551
13552 // Long Logical Instructions
13553 // TODO
13554
13555 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13556 match(Set dst (AndL src1 src2));
13557
13558 format %{ "and $dst, $src1, $src2\t# int" %}
13559
13560 ins_cost(INSN_COST);
13561 ins_encode %{
13562 __ andr(as_Register($dst$$reg),
13563 as_Register($src1$$reg),
13564 as_Register($src2$$reg));
13565 %}
13566
13567 ins_pipe(ialu_reg_reg);
13568 %}
13569
13570 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13571 match(Set dst (AndL src1 src2));
13572
13573 format %{ "and $dst, $src1, $src2\t# int" %}
13574
13575 ins_cost(INSN_COST);
13576 ins_encode %{
13577 __ andr(as_Register($dst$$reg),
13578 as_Register($src1$$reg),
13579 (uint64_t)($src2$$constant));
13580 %}
13581
13582 ins_pipe(ialu_reg_imm);
13583 %}
13584
13585 // Or Instructions
13586
13587 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13588 match(Set dst (OrL src1 src2));
13589
13590 format %{ "orr $dst, $src1, $src2\t# int" %}
13591
13592 ins_cost(INSN_COST);
13593 ins_encode %{
13594 __ orr(as_Register($dst$$reg),
13595 as_Register($src1$$reg),
13596 as_Register($src2$$reg));
13597 %}
13598
13599 ins_pipe(ialu_reg_reg);
13600 %}
13601
13602 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13603 match(Set dst (OrL src1 src2));
13604
13605 format %{ "orr $dst, $src1, $src2\t# int" %}
13606
13607 ins_cost(INSN_COST);
13608 ins_encode %{
13609 __ orr(as_Register($dst$$reg),
13610 as_Register($src1$$reg),
13611 (uint64_t)($src2$$constant));
13612 %}
13613
13614 ins_pipe(ialu_reg_imm);
13615 %}
13616
13617 // Xor Instructions
13618
13619 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13620 match(Set dst (XorL src1 src2));
13621
13622 format %{ "eor $dst, $src1, $src2\t# int" %}
13623
13624 ins_cost(INSN_COST);
13625 ins_encode %{
13626 __ eor(as_Register($dst$$reg),
13627 as_Register($src1$$reg),
13628 as_Register($src2$$reg));
13629 %}
13630
13631 ins_pipe(ialu_reg_reg);
13632 %}
13633
13634 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13635 match(Set dst (XorL src1 src2));
13636
13637 ins_cost(INSN_COST);
13638 format %{ "eor $dst, $src1, $src2\t# int" %}
13639
13640 ins_encode %{
13641 __ eor(as_Register($dst$$reg),
13642 as_Register($src1$$reg),
13643 (uint64_t)($src2$$constant));
13644 %}
13645
13646 ins_pipe(ialu_reg_imm);
13647 %}
13648
13649 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13650 %{
13651 match(Set dst (ConvI2L src));
13652
13653 ins_cost(INSN_COST);
13654 format %{ "sxtw $dst, $src\t# i2l" %}
13655 ins_encode %{
13656 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13657 %}
13658 ins_pipe(ialu_reg_shift);
13659 %}
13660
13661 // this pattern occurs in bigmath arithmetic
13662 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13663 %{
13664 match(Set dst (AndL (ConvI2L src) mask));
13665
13666 ins_cost(INSN_COST);
13667 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13668 ins_encode %{
13669 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13670 %}
13671
13672 ins_pipe(ialu_reg_shift);
13673 %}
13674
13675 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13676 match(Set dst (ConvL2I src));
13677
13678 ins_cost(INSN_COST);
13679 format %{ "movw $dst, $src \t// l2i" %}
13680
13681 ins_encode %{
13682 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13683 %}
13684
13685 ins_pipe(ialu_reg);
13686 %}
13687
13688 instruct convD2F_reg(vRegF dst, vRegD src) %{
13689 match(Set dst (ConvD2F src));
13690
13691 ins_cost(INSN_COST * 5);
13692 format %{ "fcvtd $dst, $src \t// d2f" %}
13693
13694 ins_encode %{
13695 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13696 %}
13697
13698 ins_pipe(fp_d2f);
13699 %}
13700
13701 instruct convF2D_reg(vRegD dst, vRegF src) %{
13702 match(Set dst (ConvF2D src));
13703
13704 ins_cost(INSN_COST * 5);
13705 format %{ "fcvts $dst, $src \t// f2d" %}
13706
13707 ins_encode %{
13708 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13709 %}
13710
13711 ins_pipe(fp_f2d);
13712 %}
13713
13714 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13715 match(Set dst (ConvF2I src));
13716
13717 ins_cost(INSN_COST * 5);
13718 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13719
13720 ins_encode %{
13721 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13722 %}
13723
13724 ins_pipe(fp_f2i);
13725 %}
13726
13727 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13728 match(Set dst (ConvF2L src));
13729
13730 ins_cost(INSN_COST * 5);
13731 format %{ "fcvtzs $dst, $src \t// f2l" %}
13732
13733 ins_encode %{
13734 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13735 %}
13736
13737 ins_pipe(fp_f2l);
13738 %}
13739
13740 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13741 match(Set dst (ConvF2HF src));
13742 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13743 "smov $dst, $tmp\t# move result from $tmp to $dst"
13744 %}
13745 effect(TEMP tmp);
13746 ins_encode %{
13747 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13748 %}
13749 ins_pipe(pipe_slow);
13750 %}
13751
13752 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13753 match(Set dst (ConvHF2F src));
13754 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13755 "fcvt $dst, $tmp\t# convert half to single precision"
13756 %}
13757 effect(TEMP tmp);
13758 ins_encode %{
13759 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13760 %}
13761 ins_pipe(pipe_slow);
13762 %}
13763
13764 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13765 match(Set dst (ConvI2F src));
13766
13767 ins_cost(INSN_COST * 5);
13768 format %{ "scvtfws $dst, $src \t// i2f" %}
13769
13770 ins_encode %{
13771 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13772 %}
13773
13774 ins_pipe(fp_i2f);
13775 %}
13776
13777 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13778 match(Set dst (ConvL2F src));
13779
13780 ins_cost(INSN_COST * 5);
13781 format %{ "scvtfs $dst, $src \t// l2f" %}
13782
13783 ins_encode %{
13784 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13785 %}
13786
13787 ins_pipe(fp_l2f);
13788 %}
13789
13790 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13791 match(Set dst (ConvD2I src));
13792
13793 ins_cost(INSN_COST * 5);
13794 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13795
13796 ins_encode %{
13797 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13798 %}
13799
13800 ins_pipe(fp_d2i);
13801 %}
13802
13803 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13804 match(Set dst (ConvD2L src));
13805
13806 ins_cost(INSN_COST * 5);
13807 format %{ "fcvtzd $dst, $src \t// d2l" %}
13808
13809 ins_encode %{
13810 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13811 %}
13812
13813 ins_pipe(fp_d2l);
13814 %}
13815
13816 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13817 match(Set dst (ConvI2D src));
13818
13819 ins_cost(INSN_COST * 5);
13820 format %{ "scvtfwd $dst, $src \t// i2d" %}
13821
13822 ins_encode %{
13823 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13824 %}
13825
13826 ins_pipe(fp_i2d);
13827 %}
13828
13829 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13830 match(Set dst (ConvL2D src));
13831
13832 ins_cost(INSN_COST * 5);
13833 format %{ "scvtfd $dst, $src \t// l2d" %}
13834
13835 ins_encode %{
13836 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13837 %}
13838
13839 ins_pipe(fp_l2d);
13840 %}
13841
13842 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13843 %{
13844 match(Set dst (RoundD src));
13845 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13846 format %{ "java_round_double $dst,$src"%}
13847 ins_encode %{
13848 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13849 as_FloatRegister($ftmp$$reg));
13850 %}
13851 ins_pipe(pipe_slow);
13852 %}
13853
13854 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13855 %{
13856 match(Set dst (RoundF src));
13857 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13858 format %{ "java_round_float $dst,$src"%}
13859 ins_encode %{
13860 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13861 as_FloatRegister($ftmp$$reg));
13862 %}
13863 ins_pipe(pipe_slow);
13864 %}
13865
13866 // stack <-> reg and reg <-> reg shuffles with no conversion
13867
13868 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13869
13870 match(Set dst (MoveF2I src));
13871
13872 effect(DEF dst, USE src);
13873
13874 ins_cost(4 * INSN_COST);
13875
13876 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13877
13878 ins_encode %{
13879 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13880 %}
13881
13882 ins_pipe(iload_reg_reg);
13883
13884 %}
13885
13886 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13887
13888 match(Set dst (MoveI2F src));
13889
13890 effect(DEF dst, USE src);
13891
13892 ins_cost(4 * INSN_COST);
13893
13894 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13895
13896 ins_encode %{
13897 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13898 %}
13899
13900 ins_pipe(pipe_class_memory);
13901
13902 %}
13903
13904 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13905
13906 match(Set dst (MoveD2L src));
13907
13908 effect(DEF dst, USE src);
13909
13910 ins_cost(4 * INSN_COST);
13911
13912 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13913
13914 ins_encode %{
13915 __ ldr($dst$$Register, Address(sp, $src$$disp));
13916 %}
13917
13918 ins_pipe(iload_reg_reg);
13919
13920 %}
13921
13922 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13923
13924 match(Set dst (MoveL2D src));
13925
13926 effect(DEF dst, USE src);
13927
13928 ins_cost(4 * INSN_COST);
13929
13930 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13931
13932 ins_encode %{
13933 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13934 %}
13935
13936 ins_pipe(pipe_class_memory);
13937
13938 %}
13939
13940 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13941
13942 match(Set dst (MoveF2I src));
13943
13944 effect(DEF dst, USE src);
13945
13946 ins_cost(INSN_COST);
13947
13948 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13949
13950 ins_encode %{
13951 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13952 %}
13953
13954 ins_pipe(pipe_class_memory);
13955
13956 %}
13957
13958 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13959
13960 match(Set dst (MoveI2F src));
13961
13962 effect(DEF dst, USE src);
13963
13964 ins_cost(INSN_COST);
13965
13966 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13967
13968 ins_encode %{
13969 __ strw($src$$Register, Address(sp, $dst$$disp));
13970 %}
13971
13972 ins_pipe(istore_reg_reg);
13973
13974 %}
13975
13976 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13977
13978 match(Set dst (MoveD2L src));
13979
13980 effect(DEF dst, USE src);
13981
13982 ins_cost(INSN_COST);
13983
13984 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13985
13986 ins_encode %{
13987 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13988 %}
13989
13990 ins_pipe(pipe_class_memory);
13991
13992 %}
13993
13994 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13995
13996 match(Set dst (MoveL2D src));
13997
13998 effect(DEF dst, USE src);
13999
14000 ins_cost(INSN_COST);
14001
14002 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14003
14004 ins_encode %{
14005 __ str($src$$Register, Address(sp, $dst$$disp));
14006 %}
14007
14008 ins_pipe(istore_reg_reg);
14009
14010 %}
14011
14012 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14013
14014 match(Set dst (MoveF2I src));
14015
14016 effect(DEF dst, USE src);
14017
14018 ins_cost(INSN_COST);
14019
14020 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14021
14022 ins_encode %{
14023 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14024 %}
14025
14026 ins_pipe(fp_f2i);
14027
14028 %}
14029
14030 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14031
14032 match(Set dst (MoveI2F src));
14033
14034 effect(DEF dst, USE src);
14035
14036 ins_cost(INSN_COST);
14037
14038 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14039
14040 ins_encode %{
14041 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14042 %}
14043
14044 ins_pipe(fp_i2f);
14045
14046 %}
14047
14048 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14049
14050 match(Set dst (MoveD2L src));
14051
14052 effect(DEF dst, USE src);
14053
14054 ins_cost(INSN_COST);
14055
14056 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14057
14058 ins_encode %{
14059 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14060 %}
14061
14062 ins_pipe(fp_d2l);
14063
14064 %}
14065
14066 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14067
14068 match(Set dst (MoveL2D src));
14069
14070 effect(DEF dst, USE src);
14071
14072 ins_cost(INSN_COST);
14073
14074 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14075
14076 ins_encode %{
14077 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14078 %}
14079
14080 ins_pipe(fp_l2d);
14081
14082 %}
14083
14084 // ============================================================================
14085 // clearing of an array
14086
14087 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14088 %{
14089 match(Set dummy (ClearArray cnt base));
14090 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14091
14092 ins_cost(4 * INSN_COST);
14093 format %{ "ClearArray $cnt, $base" %}
14094
14095 ins_encode %{
14096 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14097 if (tpc == nullptr) {
14098 ciEnv::current()->record_failure("CodeCache is full");
14099 return;
14100 }
14101 %}
14102
14103 ins_pipe(pipe_class_memory);
14104 %}
14105
14106 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14107 %{
14108 predicate((uint64_t)n->in(2)->get_long()
14109 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14110 match(Set dummy (ClearArray cnt base));
14111 effect(TEMP temp, USE_KILL base, KILL cr);
14112
14113 ins_cost(4 * INSN_COST);
14114 format %{ "ClearArray $cnt, $base" %}
14115
14116 ins_encode %{
14117 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14118 if (tpc == nullptr) {
14119 ciEnv::current()->record_failure("CodeCache is full");
14120 return;
14121 }
14122 %}
14123
14124 ins_pipe(pipe_class_memory);
14125 %}
14126
14127 // ============================================================================
14128 // Overflow Math Instructions
14129
14130 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14131 %{
14132 match(Set cr (OverflowAddI op1 op2));
14133
14134 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14135 ins_cost(INSN_COST);
14136 ins_encode %{
14137 __ cmnw($op1$$Register, $op2$$Register);
14138 %}
14139
14140 ins_pipe(icmp_reg_reg);
14141 %}
14142
14143 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14144 %{
14145 match(Set cr (OverflowAddI op1 op2));
14146
14147 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14148 ins_cost(INSN_COST);
14149 ins_encode %{
14150 __ cmnw($op1$$Register, $op2$$constant);
14151 %}
14152
14153 ins_pipe(icmp_reg_imm);
14154 %}
14155
14156 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14157 %{
14158 match(Set cr (OverflowAddL op1 op2));
14159
14160 format %{ "cmn $op1, $op2\t# overflow check long" %}
14161 ins_cost(INSN_COST);
14162 ins_encode %{
14163 __ cmn($op1$$Register, $op2$$Register);
14164 %}
14165
14166 ins_pipe(icmp_reg_reg);
14167 %}
14168
14169 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14170 %{
14171 match(Set cr (OverflowAddL op1 op2));
14172
14173 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14174 ins_cost(INSN_COST);
14175 ins_encode %{
14176 __ adds(zr, $op1$$Register, $op2$$constant);
14177 %}
14178
14179 ins_pipe(icmp_reg_imm);
14180 %}
14181
14182 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14183 %{
14184 match(Set cr (OverflowSubI op1 op2));
14185
14186 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14187 ins_cost(INSN_COST);
14188 ins_encode %{
14189 __ cmpw($op1$$Register, $op2$$Register);
14190 %}
14191
14192 ins_pipe(icmp_reg_reg);
14193 %}
14194
14195 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14196 %{
14197 match(Set cr (OverflowSubI op1 op2));
14198
14199 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14200 ins_cost(INSN_COST);
14201 ins_encode %{
14202 __ cmpw($op1$$Register, $op2$$constant);
14203 %}
14204
14205 ins_pipe(icmp_reg_imm);
14206 %}
14207
14208 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14209 %{
14210 match(Set cr (OverflowSubL op1 op2));
14211
14212 format %{ "cmp $op1, $op2\t# overflow check long" %}
14213 ins_cost(INSN_COST);
14214 ins_encode %{
14215 __ cmp($op1$$Register, $op2$$Register);
14216 %}
14217
14218 ins_pipe(icmp_reg_reg);
14219 %}
14220
14221 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14222 %{
14223 match(Set cr (OverflowSubL op1 op2));
14224
14225 format %{ "cmp $op1, $op2\t# overflow check long" %}
14226 ins_cost(INSN_COST);
14227 ins_encode %{
14228 __ subs(zr, $op1$$Register, $op2$$constant);
14229 %}
14230
14231 ins_pipe(icmp_reg_imm);
14232 %}
14233
14234 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14235 %{
14236 match(Set cr (OverflowSubI zero op1));
14237
14238 format %{ "cmpw zr, $op1\t# overflow check int" %}
14239 ins_cost(INSN_COST);
14240 ins_encode %{
14241 __ cmpw(zr, $op1$$Register);
14242 %}
14243
14244 ins_pipe(icmp_reg_imm);
14245 %}
14246
14247 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14248 %{
14249 match(Set cr (OverflowSubL zero op1));
14250
14251 format %{ "cmp zr, $op1\t# overflow check long" %}
14252 ins_cost(INSN_COST);
14253 ins_encode %{
14254 __ cmp(zr, $op1$$Register);
14255 %}
14256
14257 ins_pipe(icmp_reg_imm);
14258 %}
14259
14260 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14261 %{
14262 match(Set cr (OverflowMulI op1 op2));
14263
14264 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14265 "cmp rscratch1, rscratch1, sxtw\n\t"
14266 "movw rscratch1, #0x80000000\n\t"
14267 "cselw rscratch1, rscratch1, zr, NE\n\t"
14268 "cmpw rscratch1, #1" %}
14269 ins_cost(5 * INSN_COST);
14270 ins_encode %{
14271 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14272 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14273 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14274 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14275 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14276 %}
14277
14278 ins_pipe(pipe_slow);
14279 %}
14280
14281 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14282 %{
14283 match(If cmp (OverflowMulI op1 op2));
14284 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14285 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14286 effect(USE labl, KILL cr);
14287
14288 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14289 "cmp rscratch1, rscratch1, sxtw\n\t"
14290 "b$cmp $labl" %}
14291 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14292 ins_encode %{
14293 Label* L = $labl$$label;
14294 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14295 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14296 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14297 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14298 %}
14299
14300 ins_pipe(pipe_serial);
14301 %}
14302
14303 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14304 %{
14305 match(Set cr (OverflowMulL op1 op2));
14306
14307 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14308 "smulh rscratch2, $op1, $op2\n\t"
14309 "cmp rscratch2, rscratch1, ASR #63\n\t"
14310 "movw rscratch1, #0x80000000\n\t"
14311 "cselw rscratch1, rscratch1, zr, NE\n\t"
14312 "cmpw rscratch1, #1" %}
14313 ins_cost(6 * INSN_COST);
14314 ins_encode %{
14315 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14316 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14317 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14318 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14319 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14320 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14321 %}
14322
14323 ins_pipe(pipe_slow);
14324 %}
14325
14326 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14327 %{
14328 match(If cmp (OverflowMulL op1 op2));
14329 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14330 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14331 effect(USE labl, KILL cr);
14332
14333 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14334 "smulh rscratch2, $op1, $op2\n\t"
14335 "cmp rscratch2, rscratch1, ASR #63\n\t"
14336 "b$cmp $labl" %}
14337 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14338 ins_encode %{
14339 Label* L = $labl$$label;
14340 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14341 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14342 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14343 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14344 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14345 %}
14346
14347 ins_pipe(pipe_serial);
14348 %}
14349
14350 // ============================================================================
14351 // Compare Instructions
14352
14353 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14354 %{
14355 match(Set cr (CmpI op1 op2));
14356
14357 effect(DEF cr, USE op1, USE op2);
14358
14359 ins_cost(INSN_COST);
14360 format %{ "cmpw $op1, $op2" %}
14361
14362 ins_encode(aarch64_enc_cmpw(op1, op2));
14363
14364 ins_pipe(icmp_reg_reg);
14365 %}
14366
14367 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14368 %{
14369 match(Set cr (CmpI op1 zero));
14370
14371 effect(DEF cr, USE op1);
14372
14373 ins_cost(INSN_COST);
14374 format %{ "cmpw $op1, 0" %}
14375
14376 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14377
14378 ins_pipe(icmp_reg_imm);
14379 %}
14380
14381 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14382 %{
14383 match(Set cr (CmpI op1 op2));
14384
14385 effect(DEF cr, USE op1);
14386
14387 ins_cost(INSN_COST);
14388 format %{ "cmpw $op1, $op2" %}
14389
14390 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14391
14392 ins_pipe(icmp_reg_imm);
14393 %}
14394
14395 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14396 %{
14397 match(Set cr (CmpI op1 op2));
14398
14399 effect(DEF cr, USE op1);
14400
14401 ins_cost(INSN_COST * 2);
14402 format %{ "cmpw $op1, $op2" %}
14403
14404 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14405
14406 ins_pipe(icmp_reg_imm);
14407 %}
14408
14409 // Unsigned compare Instructions; really, same as signed compare
14410 // except it should only be used to feed an If or a CMovI which takes a
14411 // cmpOpU.
14412
14413 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14414 %{
14415 match(Set cr (CmpU op1 op2));
14416
14417 effect(DEF cr, USE op1, USE op2);
14418
14419 ins_cost(INSN_COST);
14420 format %{ "cmpw $op1, $op2\t# unsigned" %}
14421
14422 ins_encode(aarch64_enc_cmpw(op1, op2));
14423
14424 ins_pipe(icmp_reg_reg);
14425 %}
14426
14427 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14428 %{
14429 match(Set cr (CmpU op1 zero));
14430
14431 effect(DEF cr, USE op1);
14432
14433 ins_cost(INSN_COST);
14434 format %{ "cmpw $op1, #0\t# unsigned" %}
14435
14436 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14437
14438 ins_pipe(icmp_reg_imm);
14439 %}
14440
14441 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14442 %{
14443 match(Set cr (CmpU op1 op2));
14444
14445 effect(DEF cr, USE op1);
14446
14447 ins_cost(INSN_COST);
14448 format %{ "cmpw $op1, $op2\t# unsigned" %}
14449
14450 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14451
14452 ins_pipe(icmp_reg_imm);
14453 %}
14454
14455 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14456 %{
14457 match(Set cr (CmpU op1 op2));
14458
14459 effect(DEF cr, USE op1);
14460
14461 ins_cost(INSN_COST * 2);
14462 format %{ "cmpw $op1, $op2\t# unsigned" %}
14463
14464 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14465
14466 ins_pipe(icmp_reg_imm);
14467 %}
14468
14469 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14470 %{
14471 match(Set cr (CmpL op1 op2));
14472
14473 effect(DEF cr, USE op1, USE op2);
14474
14475 ins_cost(INSN_COST);
14476 format %{ "cmp $op1, $op2" %}
14477
14478 ins_encode(aarch64_enc_cmp(op1, op2));
14479
14480 ins_pipe(icmp_reg_reg);
14481 %}
14482
14483 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14484 %{
14485 match(Set cr (CmpL op1 zero));
14486
14487 effect(DEF cr, USE op1);
14488
14489 ins_cost(INSN_COST);
14490 format %{ "tst $op1" %}
14491
14492 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14493
14494 ins_pipe(icmp_reg_imm);
14495 %}
14496
14497 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14498 %{
14499 match(Set cr (CmpL op1 op2));
14500
14501 effect(DEF cr, USE op1);
14502
14503 ins_cost(INSN_COST);
14504 format %{ "cmp $op1, $op2" %}
14505
14506 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14507
14508 ins_pipe(icmp_reg_imm);
14509 %}
14510
14511 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14512 %{
14513 match(Set cr (CmpL op1 op2));
14514
14515 effect(DEF cr, USE op1);
14516
14517 ins_cost(INSN_COST * 2);
14518 format %{ "cmp $op1, $op2" %}
14519
14520 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14521
14522 ins_pipe(icmp_reg_imm);
14523 %}
14524
14525 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14526 %{
14527 match(Set cr (CmpUL op1 op2));
14528
14529 effect(DEF cr, USE op1, USE op2);
14530
14531 ins_cost(INSN_COST);
14532 format %{ "cmp $op1, $op2" %}
14533
14534 ins_encode(aarch64_enc_cmp(op1, op2));
14535
14536 ins_pipe(icmp_reg_reg);
14537 %}
14538
14539 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14540 %{
14541 match(Set cr (CmpUL op1 zero));
14542
14543 effect(DEF cr, USE op1);
14544
14545 ins_cost(INSN_COST);
14546 format %{ "tst $op1" %}
14547
14548 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14549
14550 ins_pipe(icmp_reg_imm);
14551 %}
14552
14553 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14554 %{
14555 match(Set cr (CmpUL op1 op2));
14556
14557 effect(DEF cr, USE op1);
14558
14559 ins_cost(INSN_COST);
14560 format %{ "cmp $op1, $op2" %}
14561
14562 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14563
14564 ins_pipe(icmp_reg_imm);
14565 %}
14566
14567 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14568 %{
14569 match(Set cr (CmpUL op1 op2));
14570
14571 effect(DEF cr, USE op1);
14572
14573 ins_cost(INSN_COST * 2);
14574 format %{ "cmp $op1, $op2" %}
14575
14576 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14577
14578 ins_pipe(icmp_reg_imm);
14579 %}
14580
14581 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14582 %{
14583 match(Set cr (CmpP op1 op2));
14584
14585 effect(DEF cr, USE op1, USE op2);
14586
14587 ins_cost(INSN_COST);
14588 format %{ "cmp $op1, $op2\t // ptr" %}
14589
14590 ins_encode(aarch64_enc_cmpp(op1, op2));
14591
14592 ins_pipe(icmp_reg_reg);
14593 %}
14594
14595 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14596 %{
14597 match(Set cr (CmpN op1 op2));
14598
14599 effect(DEF cr, USE op1, USE op2);
14600
14601 ins_cost(INSN_COST);
14602 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14603
14604 ins_encode(aarch64_enc_cmpn(op1, op2));
14605
14606 ins_pipe(icmp_reg_reg);
14607 %}
14608
14609 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14610 %{
14611 match(Set cr (CmpP op1 zero));
14612
14613 effect(DEF cr, USE op1, USE zero);
14614
14615 ins_cost(INSN_COST);
14616 format %{ "cmp $op1, 0\t // ptr" %}
14617
14618 ins_encode(aarch64_enc_testp(op1));
14619
14620 ins_pipe(icmp_reg_imm);
14621 %}
14622
14623 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14624 %{
14625 match(Set cr (CmpN op1 zero));
14626
14627 effect(DEF cr, USE op1, USE zero);
14628
14629 ins_cost(INSN_COST);
14630 format %{ "cmp $op1, 0\t // compressed ptr" %}
14631
14632 ins_encode(aarch64_enc_testn(op1));
14633
14634 ins_pipe(icmp_reg_imm);
14635 %}
14636
14637 // FP comparisons
14638 //
14639 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14640 // using normal cmpOp. See declaration of rFlagsReg for details.
14641
14642 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14643 %{
14644 match(Set cr (CmpF src1 src2));
14645
14646 ins_cost(3 * INSN_COST);
14647 format %{ "fcmps $src1, $src2" %}
14648
14649 ins_encode %{
14650 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14651 %}
14652
14653 ins_pipe(pipe_class_compare);
14654 %}
14655
14656 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14657 %{
14658 match(Set cr (CmpF src1 src2));
14659
14660 ins_cost(3 * INSN_COST);
14661 format %{ "fcmps $src1, 0.0" %}
14662
14663 ins_encode %{
14664 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14665 %}
14666
14667 ins_pipe(pipe_class_compare);
14668 %}
14669 // FROM HERE
14670
14671 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14672 %{
14673 match(Set cr (CmpD src1 src2));
14674
14675 ins_cost(3 * INSN_COST);
14676 format %{ "fcmpd $src1, $src2" %}
14677
14678 ins_encode %{
14679 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14680 %}
14681
14682 ins_pipe(pipe_class_compare);
14683 %}
14684
14685 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14686 %{
14687 match(Set cr (CmpD src1 src2));
14688
14689 ins_cost(3 * INSN_COST);
14690 format %{ "fcmpd $src1, 0.0" %}
14691
14692 ins_encode %{
14693 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14694 %}
14695
14696 ins_pipe(pipe_class_compare);
14697 %}
14698
14699 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14700 %{
14701 match(Set dst (CmpF3 src1 src2));
14702 effect(KILL cr);
14703
14704 ins_cost(5 * INSN_COST);
14705 format %{ "fcmps $src1, $src2\n\t"
14706 "csinvw($dst, zr, zr, eq\n\t"
14707 "csnegw($dst, $dst, $dst, lt)"
14708 %}
14709
14710 ins_encode %{
14711 Label done;
14712 FloatRegister s1 = as_FloatRegister($src1$$reg);
14713 FloatRegister s2 = as_FloatRegister($src2$$reg);
14714 Register d = as_Register($dst$$reg);
14715 __ fcmps(s1, s2);
14716 // installs 0 if EQ else -1
14717 __ csinvw(d, zr, zr, Assembler::EQ);
14718 // keeps -1 if less or unordered else installs 1
14719 __ csnegw(d, d, d, Assembler::LT);
14720 __ bind(done);
14721 %}
14722
14723 ins_pipe(pipe_class_default);
14724
14725 %}
14726
14727 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14728 %{
14729 match(Set dst (CmpD3 src1 src2));
14730 effect(KILL cr);
14731
14732 ins_cost(5 * INSN_COST);
14733 format %{ "fcmpd $src1, $src2\n\t"
14734 "csinvw($dst, zr, zr, eq\n\t"
14735 "csnegw($dst, $dst, $dst, lt)"
14736 %}
14737
14738 ins_encode %{
14739 Label done;
14740 FloatRegister s1 = as_FloatRegister($src1$$reg);
14741 FloatRegister s2 = as_FloatRegister($src2$$reg);
14742 Register d = as_Register($dst$$reg);
14743 __ fcmpd(s1, s2);
14744 // installs 0 if EQ else -1
14745 __ csinvw(d, zr, zr, Assembler::EQ);
14746 // keeps -1 if less or unordered else installs 1
14747 __ csnegw(d, d, d, Assembler::LT);
14748 __ bind(done);
14749 %}
14750 ins_pipe(pipe_class_default);
14751
14752 %}
14753
14754 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14755 %{
14756 match(Set dst (CmpF3 src1 zero));
14757 effect(KILL cr);
14758
14759 ins_cost(5 * INSN_COST);
14760 format %{ "fcmps $src1, 0.0\n\t"
14761 "csinvw($dst, zr, zr, eq\n\t"
14762 "csnegw($dst, $dst, $dst, lt)"
14763 %}
14764
14765 ins_encode %{
14766 Label done;
14767 FloatRegister s1 = as_FloatRegister($src1$$reg);
14768 Register d = as_Register($dst$$reg);
14769 __ fcmps(s1, 0.0);
14770 // installs 0 if EQ else -1
14771 __ csinvw(d, zr, zr, Assembler::EQ);
14772 // keeps -1 if less or unordered else installs 1
14773 __ csnegw(d, d, d, Assembler::LT);
14774 __ bind(done);
14775 %}
14776
14777 ins_pipe(pipe_class_default);
14778
14779 %}
14780
14781 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14782 %{
14783 match(Set dst (CmpD3 src1 zero));
14784 effect(KILL cr);
14785
14786 ins_cost(5 * INSN_COST);
14787 format %{ "fcmpd $src1, 0.0\n\t"
14788 "csinvw($dst, zr, zr, eq\n\t"
14789 "csnegw($dst, $dst, $dst, lt)"
14790 %}
14791
14792 ins_encode %{
14793 Label done;
14794 FloatRegister s1 = as_FloatRegister($src1$$reg);
14795 Register d = as_Register($dst$$reg);
14796 __ fcmpd(s1, 0.0);
14797 // installs 0 if EQ else -1
14798 __ csinvw(d, zr, zr, Assembler::EQ);
14799 // keeps -1 if less or unordered else installs 1
14800 __ csnegw(d, d, d, Assembler::LT);
14801 __ bind(done);
14802 %}
14803 ins_pipe(pipe_class_default);
14804
14805 %}
14806
14807 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14808 %{
14809 match(Set dst (CmpLTMask p q));
14810 effect(KILL cr);
14811
14812 ins_cost(3 * INSN_COST);
14813
14814 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14815 "csetw $dst, lt\n\t"
14816 "subw $dst, zr, $dst"
14817 %}
14818
14819 ins_encode %{
14820 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14821 __ csetw(as_Register($dst$$reg), Assembler::LT);
14822 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14823 %}
14824
14825 ins_pipe(ialu_reg_reg);
14826 %}
14827
14828 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14829 %{
14830 match(Set dst (CmpLTMask src zero));
14831 effect(KILL cr);
14832
14833 ins_cost(INSN_COST);
14834
14835 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14836
14837 ins_encode %{
14838 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14839 %}
14840
14841 ins_pipe(ialu_reg_shift);
14842 %}
14843
14844 // ============================================================================
14845 // Max and Min
14846
14847 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14848
14849 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14850 %{
14851 effect(DEF cr, USE src);
14852 ins_cost(INSN_COST);
14853 format %{ "cmpw $src, 0" %}
14854
14855 ins_encode %{
14856 __ cmpw($src$$Register, 0);
14857 %}
14858 ins_pipe(icmp_reg_imm);
14859 %}
14860
14861 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14862 %{
14863 match(Set dst (MinI src1 src2));
14864 ins_cost(INSN_COST * 3);
14865
14866 expand %{
14867 rFlagsReg cr;
14868 compI_reg_reg(cr, src1, src2);
14869 cmovI_reg_reg_lt(dst, src1, src2, cr);
14870 %}
14871 %}
14872
14873 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14874 %{
14875 match(Set dst (MaxI src1 src2));
14876 ins_cost(INSN_COST * 3);
14877
14878 expand %{
14879 rFlagsReg cr;
14880 compI_reg_reg(cr, src1, src2);
14881 cmovI_reg_reg_gt(dst, src1, src2, cr);
14882 %}
14883 %}
14884
14885
14886 // ============================================================================
14887 // Branch Instructions
14888
14889 // Direct Branch.
14890 instruct branch(label lbl)
14891 %{
14892 match(Goto);
14893
14894 effect(USE lbl);
14895
14896 ins_cost(BRANCH_COST);
14897 format %{ "b $lbl" %}
14898
14899 ins_encode(aarch64_enc_b(lbl));
14900
14901 ins_pipe(pipe_branch);
14902 %}
14903
14904 // Conditional Near Branch
14905 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14906 %{
14907 // Same match rule as `branchConFar'.
14908 match(If cmp cr);
14909
14910 effect(USE lbl);
14911
14912 ins_cost(BRANCH_COST);
14913 // If set to 1 this indicates that the current instruction is a
14914 // short variant of a long branch. This avoids using this
14915 // instruction in first-pass matching. It will then only be used in
14916 // the `Shorten_branches' pass.
14917 // ins_short_branch(1);
14918 format %{ "b$cmp $lbl" %}
14919
14920 ins_encode(aarch64_enc_br_con(cmp, lbl));
14921
14922 ins_pipe(pipe_branch_cond);
14923 %}
14924
14925 // Conditional Near Branch Unsigned
14926 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14927 %{
14928 // Same match rule as `branchConFar'.
14929 match(If cmp cr);
14930
14931 effect(USE lbl);
14932
14933 ins_cost(BRANCH_COST);
14934 // If set to 1 this indicates that the current instruction is a
14935 // short variant of a long branch. This avoids using this
14936 // instruction in first-pass matching. It will then only be used in
14937 // the `Shorten_branches' pass.
14938 // ins_short_branch(1);
14939 format %{ "b$cmp $lbl\t# unsigned" %}
14940
14941 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14942
14943 ins_pipe(pipe_branch_cond);
14944 %}
14945
14946 // Make use of CBZ and CBNZ. These instructions, as well as being
14947 // shorter than (cmp; branch), have the additional benefit of not
14948 // killing the flags.
14949
14950 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14951 match(If cmp (CmpI op1 op2));
14952 effect(USE labl);
14953
14954 ins_cost(BRANCH_COST);
14955 format %{ "cbw$cmp $op1, $labl" %}
14956 ins_encode %{
14957 Label* L = $labl$$label;
14958 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14959 if (cond == Assembler::EQ)
14960 __ cbzw($op1$$Register, *L);
14961 else
14962 __ cbnzw($op1$$Register, *L);
14963 %}
14964 ins_pipe(pipe_cmp_branch);
14965 %}
14966
14967 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14968 match(If cmp (CmpL op1 op2));
14969 effect(USE labl);
14970
14971 ins_cost(BRANCH_COST);
14972 format %{ "cb$cmp $op1, $labl" %}
14973 ins_encode %{
14974 Label* L = $labl$$label;
14975 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14976 if (cond == Assembler::EQ)
14977 __ cbz($op1$$Register, *L);
14978 else
14979 __ cbnz($op1$$Register, *L);
14980 %}
14981 ins_pipe(pipe_cmp_branch);
14982 %}
14983
14984 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14985 match(If cmp (CmpP op1 op2));
14986 effect(USE labl);
14987
14988 ins_cost(BRANCH_COST);
14989 format %{ "cb$cmp $op1, $labl" %}
14990 ins_encode %{
14991 Label* L = $labl$$label;
14992 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14993 if (cond == Assembler::EQ)
14994 __ cbz($op1$$Register, *L);
14995 else
14996 __ cbnz($op1$$Register, *L);
14997 %}
14998 ins_pipe(pipe_cmp_branch);
14999 %}
15000
15001 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15002 match(If cmp (CmpN op1 op2));
15003 effect(USE labl);
15004
15005 ins_cost(BRANCH_COST);
15006 format %{ "cbw$cmp $op1, $labl" %}
15007 ins_encode %{
15008 Label* L = $labl$$label;
15009 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15010 if (cond == Assembler::EQ)
15011 __ cbzw($op1$$Register, *L);
15012 else
15013 __ cbnzw($op1$$Register, *L);
15014 %}
15015 ins_pipe(pipe_cmp_branch);
15016 %}
15017
15018 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15019 match(If cmp (CmpP (DecodeN oop) zero));
15020 effect(USE labl);
15021
15022 ins_cost(BRANCH_COST);
15023 format %{ "cb$cmp $oop, $labl" %}
15024 ins_encode %{
15025 Label* L = $labl$$label;
15026 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15027 if (cond == Assembler::EQ)
15028 __ cbzw($oop$$Register, *L);
15029 else
15030 __ cbnzw($oop$$Register, *L);
15031 %}
15032 ins_pipe(pipe_cmp_branch);
15033 %}
15034
15035 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15036 match(If cmp (CmpU op1 op2));
15037 effect(USE labl);
15038
15039 ins_cost(BRANCH_COST);
15040 format %{ "cbw$cmp $op1, $labl" %}
15041 ins_encode %{
15042 Label* L = $labl$$label;
15043 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15044 if (cond == Assembler::EQ || cond == Assembler::LS) {
15045 __ cbzw($op1$$Register, *L);
15046 } else {
15047 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15048 __ cbnzw($op1$$Register, *L);
15049 }
15050 %}
15051 ins_pipe(pipe_cmp_branch);
15052 %}
15053
15054 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15055 match(If cmp (CmpUL op1 op2));
15056 effect(USE labl);
15057
15058 ins_cost(BRANCH_COST);
15059 format %{ "cb$cmp $op1, $labl" %}
15060 ins_encode %{
15061 Label* L = $labl$$label;
15062 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15063 if (cond == Assembler::EQ || cond == Assembler::LS) {
15064 __ cbz($op1$$Register, *L);
15065 } else {
15066 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15067 __ cbnz($op1$$Register, *L);
15068 }
15069 %}
15070 ins_pipe(pipe_cmp_branch);
15071 %}
15072
15073 // Test bit and Branch
15074
15075 // Patterns for short (< 32KiB) variants
15076 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15077 match(If cmp (CmpL op1 op2));
15078 effect(USE labl);
15079
15080 ins_cost(BRANCH_COST);
15081 format %{ "cb$cmp $op1, $labl # long" %}
15082 ins_encode %{
15083 Label* L = $labl$$label;
15084 Assembler::Condition cond =
15085 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15086 __ tbr(cond, $op1$$Register, 63, *L);
15087 %}
15088 ins_pipe(pipe_cmp_branch);
15089 ins_short_branch(1);
15090 %}
15091
15092 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15093 match(If cmp (CmpI op1 op2));
15094 effect(USE labl);
15095
15096 ins_cost(BRANCH_COST);
15097 format %{ "cb$cmp $op1, $labl # int" %}
15098 ins_encode %{
15099 Label* L = $labl$$label;
15100 Assembler::Condition cond =
15101 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15102 __ tbr(cond, $op1$$Register, 31, *L);
15103 %}
15104 ins_pipe(pipe_cmp_branch);
15105 ins_short_branch(1);
15106 %}
15107
15108 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15109 match(If cmp (CmpL (AndL op1 op2) op3));
15110 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15111 effect(USE labl);
15112
15113 ins_cost(BRANCH_COST);
15114 format %{ "tb$cmp $op1, $op2, $labl" %}
15115 ins_encode %{
15116 Label* L = $labl$$label;
15117 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15118 int bit = exact_log2_long($op2$$constant);
15119 __ tbr(cond, $op1$$Register, bit, *L);
15120 %}
15121 ins_pipe(pipe_cmp_branch);
15122 ins_short_branch(1);
15123 %}
15124
15125 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15126 match(If cmp (CmpI (AndI op1 op2) op3));
15127 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15128 effect(USE labl);
15129
15130 ins_cost(BRANCH_COST);
15131 format %{ "tb$cmp $op1, $op2, $labl" %}
15132 ins_encode %{
15133 Label* L = $labl$$label;
15134 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15135 int bit = exact_log2((juint)$op2$$constant);
15136 __ tbr(cond, $op1$$Register, bit, *L);
15137 %}
15138 ins_pipe(pipe_cmp_branch);
15139 ins_short_branch(1);
15140 %}
15141
15142 // And far variants
15143 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15144 match(If cmp (CmpL op1 op2));
15145 effect(USE labl);
15146
15147 ins_cost(BRANCH_COST);
15148 format %{ "cb$cmp $op1, $labl # long" %}
15149 ins_encode %{
15150 Label* L = $labl$$label;
15151 Assembler::Condition cond =
15152 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15153 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15154 %}
15155 ins_pipe(pipe_cmp_branch);
15156 %}
15157
15158 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15159 match(If cmp (CmpI op1 op2));
15160 effect(USE labl);
15161
15162 ins_cost(BRANCH_COST);
15163 format %{ "cb$cmp $op1, $labl # int" %}
15164 ins_encode %{
15165 Label* L = $labl$$label;
15166 Assembler::Condition cond =
15167 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15168 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15169 %}
15170 ins_pipe(pipe_cmp_branch);
15171 %}
15172
15173 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15174 match(If cmp (CmpL (AndL op1 op2) op3));
15175 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15176 effect(USE labl);
15177
15178 ins_cost(BRANCH_COST);
15179 format %{ "tb$cmp $op1, $op2, $labl" %}
15180 ins_encode %{
15181 Label* L = $labl$$label;
15182 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15183 int bit = exact_log2_long($op2$$constant);
15184 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15185 %}
15186 ins_pipe(pipe_cmp_branch);
15187 %}
15188
15189 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15190 match(If cmp (CmpI (AndI op1 op2) op3));
15191 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15192 effect(USE labl);
15193
15194 ins_cost(BRANCH_COST);
15195 format %{ "tb$cmp $op1, $op2, $labl" %}
15196 ins_encode %{
15197 Label* L = $labl$$label;
15198 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15199 int bit = exact_log2((juint)$op2$$constant);
15200 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15201 %}
15202 ins_pipe(pipe_cmp_branch);
15203 %}
15204
15205 // Test bits
15206
15207 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15208 match(Set cr (CmpL (AndL op1 op2) op3));
15209 predicate(Assembler::operand_valid_for_logical_immediate
15210 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15211
15212 ins_cost(INSN_COST);
15213 format %{ "tst $op1, $op2 # long" %}
15214 ins_encode %{
15215 __ tst($op1$$Register, $op2$$constant);
15216 %}
15217 ins_pipe(ialu_reg_reg);
15218 %}
15219
15220 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15221 match(Set cr (CmpI (AndI op1 op2) op3));
15222 predicate(Assembler::operand_valid_for_logical_immediate
15223 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15224
15225 ins_cost(INSN_COST);
15226 format %{ "tst $op1, $op2 # int" %}
15227 ins_encode %{
15228 __ tstw($op1$$Register, $op2$$constant);
15229 %}
15230 ins_pipe(ialu_reg_reg);
15231 %}
15232
15233 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15234 match(Set cr (CmpL (AndL op1 op2) op3));
15235
15236 ins_cost(INSN_COST);
15237 format %{ "tst $op1, $op2 # long" %}
15238 ins_encode %{
15239 __ tst($op1$$Register, $op2$$Register);
15240 %}
15241 ins_pipe(ialu_reg_reg);
15242 %}
15243
15244 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15245 match(Set cr (CmpI (AndI op1 op2) op3));
15246
15247 ins_cost(INSN_COST);
15248 format %{ "tstw $op1, $op2 # int" %}
15249 ins_encode %{
15250 __ tstw($op1$$Register, $op2$$Register);
15251 %}
15252 ins_pipe(ialu_reg_reg);
15253 %}
15254
15255
15256 // Conditional Far Branch
15257 // Conditional Far Branch Unsigned
15258 // TODO: fixme
15259
15260 // counted loop end branch near
15261 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15262 %{
15263 match(CountedLoopEnd cmp cr);
15264
15265 effect(USE lbl);
15266
15267 ins_cost(BRANCH_COST);
15268 // short variant.
15269 // ins_short_branch(1);
15270 format %{ "b$cmp $lbl \t// counted loop end" %}
15271
15272 ins_encode(aarch64_enc_br_con(cmp, lbl));
15273
15274 ins_pipe(pipe_branch);
15275 %}
15276
15277 // counted loop end branch far
15278 // TODO: fixme
15279
15280 // ============================================================================
15281 // inlined locking and unlocking
15282
15283 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15284 %{
15285 match(Set cr (FastLock object box));
15286 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15287
15288 ins_cost(5 * INSN_COST);
15289 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15290
15291 ins_encode %{
15292 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15293 %}
15294
15295 ins_pipe(pipe_serial);
15296 %}
15297
15298 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15299 %{
15300 match(Set cr (FastUnlock object box));
15301 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15302
15303 ins_cost(5 * INSN_COST);
15304 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15305
15306 ins_encode %{
15307 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15308 %}
15309
15310 ins_pipe(pipe_serial);
15311 %}
15312
15313 // ============================================================================
15314 // Safepoint Instructions
15315
15316 // TODO
15317 // provide a near and far version of this code
15318
15319 instruct safePoint(rFlagsReg cr, iRegP poll)
15320 %{
15321 match(SafePoint poll);
15322 effect(KILL cr);
15323
15324 format %{
15325 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15326 %}
15327 ins_encode %{
15328 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15329 %}
15330 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15331 %}
15332
15333
15334 // ============================================================================
15335 // Procedure Call/Return Instructions
15336
15337 // Call Java Static Instruction
15338
15339 instruct CallStaticJavaDirect(method meth)
15340 %{
15341 match(CallStaticJava);
15342
15343 effect(USE meth);
15344
15345 ins_cost(CALL_COST);
15346
15347 format %{ "call,static $meth \t// ==> " %}
15348
15349 ins_encode(aarch64_enc_java_static_call(meth),
15350 aarch64_enc_call_epilog);
15351
15352 ins_pipe(pipe_class_call);
15353 %}
15354
15355 // TO HERE
15356
15357 // Call Java Dynamic Instruction
15358 instruct CallDynamicJavaDirect(method meth)
15359 %{
15360 match(CallDynamicJava);
15361
15362 effect(USE meth);
15363
15364 ins_cost(CALL_COST);
15365
15366 format %{ "CALL,dynamic $meth \t// ==> " %}
15367
15368 ins_encode(aarch64_enc_java_dynamic_call(meth),
15369 aarch64_enc_call_epilog);
15370
15371 ins_pipe(pipe_class_call);
15372 %}
15373
15374 // Call Runtime Instruction
15375
15376 instruct CallRuntimeDirect(method meth)
15377 %{
15378 match(CallRuntime);
15379
15380 effect(USE meth);
15381
15382 ins_cost(CALL_COST);
15383
15384 format %{ "CALL, runtime $meth" %}
15385
15386 ins_encode( aarch64_enc_java_to_runtime(meth) );
15387
15388 ins_pipe(pipe_class_call);
15389 %}
15390
15391 // Call Runtime Instruction
15392
15393 instruct CallLeafDirect(method meth)
15394 %{
15395 match(CallLeaf);
15396
15397 effect(USE meth);
15398
15399 ins_cost(CALL_COST);
15400
15401 format %{ "CALL, runtime leaf $meth" %}
15402
15403 ins_encode( aarch64_enc_java_to_runtime(meth) );
15404
15405 ins_pipe(pipe_class_call);
15406 %}
15407
15408 // Call Runtime Instruction without safepoint and with vector arguments
15409 instruct CallLeafDirectVector(method meth)
15410 %{
15411 match(CallLeafVector);
15412
15413 effect(USE meth);
15414
15415 ins_cost(CALL_COST);
15416
15417 format %{ "CALL, runtime leaf vector $meth" %}
15418
15419 ins_encode(aarch64_enc_java_to_runtime(meth));
15420
15421 ins_pipe(pipe_class_call);
15422 %}
15423
15424 // Call Runtime Instruction
15425
15426 instruct CallLeafNoFPDirect(method meth)
15427 %{
15428 match(CallLeafNoFP);
15429
15430 effect(USE meth);
15431
15432 ins_cost(CALL_COST);
15433
15434 format %{ "CALL, runtime leaf nofp $meth" %}
15435
15436 ins_encode( aarch64_enc_java_to_runtime(meth) );
15437
15438 ins_pipe(pipe_class_call);
15439 %}
15440
15441 // Tail Call; Jump from runtime stub to Java code.
15442 // Also known as an 'interprocedural jump'.
15443 // Target of jump will eventually return to caller.
15444 // TailJump below removes the return address.
15445 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15446 // emitted just above the TailCall which has reset rfp to the caller state.
15447 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15448 %{
15449 match(TailCall jump_target method_ptr);
15450
15451 ins_cost(CALL_COST);
15452
15453 format %{ "br $jump_target\t# $method_ptr holds method" %}
15454
15455 ins_encode(aarch64_enc_tail_call(jump_target));
15456
15457 ins_pipe(pipe_class_call);
15458 %}
15459
15460 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15461 %{
15462 match(TailJump jump_target ex_oop);
15463
15464 ins_cost(CALL_COST);
15465
15466 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15467
15468 ins_encode(aarch64_enc_tail_jmp(jump_target));
15469
15470 ins_pipe(pipe_class_call);
15471 %}
15472
15473 // Forward exception.
15474 instruct ForwardExceptionjmp()
15475 %{
15476 match(ForwardException);
15477 ins_cost(CALL_COST);
15478
15479 format %{ "b forward_exception_stub" %}
15480 ins_encode %{
15481 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15482 %}
15483 ins_pipe(pipe_class_call);
15484 %}
15485
15486 // Create exception oop: created by stack-crawling runtime code.
15487 // Created exception is now available to this handler, and is setup
15488 // just prior to jumping to this handler. No code emitted.
15489 // TODO check
15490 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15491 instruct CreateException(iRegP_R0 ex_oop)
15492 %{
15493 match(Set ex_oop (CreateEx));
15494
15495 format %{ " -- \t// exception oop; no code emitted" %}
15496
15497 size(0);
15498
15499 ins_encode( /*empty*/ );
15500
15501 ins_pipe(pipe_class_empty);
15502 %}
15503
15504 // Rethrow exception: The exception oop will come in the first
15505 // argument position. Then JUMP (not call) to the rethrow stub code.
15506 instruct RethrowException() %{
15507 match(Rethrow);
15508 ins_cost(CALL_COST);
15509
15510 format %{ "b rethrow_stub" %}
15511
15512 ins_encode( aarch64_enc_rethrow() );
15513
15514 ins_pipe(pipe_class_call);
15515 %}
15516
15517
15518 // Return Instruction
15519 // epilog node loads ret address into lr as part of frame pop
15520 instruct Ret()
15521 %{
15522 match(Return);
15523
15524 format %{ "ret\t// return register" %}
15525
15526 ins_encode( aarch64_enc_ret() );
15527
15528 ins_pipe(pipe_branch);
15529 %}
15530
15531 // Die now.
15532 instruct ShouldNotReachHere() %{
15533 match(Halt);
15534
15535 ins_cost(CALL_COST);
15536 format %{ "ShouldNotReachHere" %}
15537
15538 ins_encode %{
15539 if (is_reachable()) {
15540 const char* str = __ code_string(_halt_reason);
15541 __ stop(str);
15542 }
15543 %}
15544
15545 ins_pipe(pipe_class_default);
15546 %}
15547
15548 // ============================================================================
15549 // Partial Subtype Check
15550 //
15551 // superklass array for an instance of the superklass. Set a hidden
15552 // internal cache on a hit (cache is checked with exposed code in
15553 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15554 // encoding ALSO sets flags.
15555
15556 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15557 %{
15558 match(Set result (PartialSubtypeCheck sub super));
15559 predicate(!UseSecondarySupersTable);
15560 effect(KILL cr, KILL temp);
15561
15562 ins_cost(20 * INSN_COST); // slightly larger than the next version
15563 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15564
15565 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15566
15567 opcode(0x1); // Force zero of result reg on hit
15568
15569 ins_pipe(pipe_class_memory);
15570 %}
15571
15572 // Two versions of partialSubtypeCheck, both used when we need to
15573 // search for a super class in the secondary supers array. The first
15574 // is used when we don't know _a priori_ the class being searched
15575 // for. The second, far more common, is used when we do know: this is
15576 // used for instanceof, checkcast, and any case where C2 can determine
15577 // it by constant propagation.
15578
15579 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15580 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15581 rFlagsReg cr)
15582 %{
15583 match(Set result (PartialSubtypeCheck sub super));
15584 predicate(UseSecondarySupersTable);
15585 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15586
15587 ins_cost(10 * INSN_COST); // slightly larger than the next version
15588 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15589
15590 ins_encode %{
15591 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15592 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15593 $vtemp$$FloatRegister,
15594 $result$$Register, /*L_success*/nullptr);
15595 %}
15596
15597 ins_pipe(pipe_class_memory);
15598 %}
15599
15600 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15601 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15602 rFlagsReg cr)
15603 %{
15604 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15605 predicate(UseSecondarySupersTable);
15606 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15607
15608 ins_cost(5 * INSN_COST); // smaller than the next version
15609 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15610
15611 ins_encode %{
15612 bool success = false;
15613 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15614 if (InlineSecondarySupersTest) {
15615 success =
15616 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15617 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15618 $vtemp$$FloatRegister,
15619 $result$$Register,
15620 super_klass_slot);
15621 } else {
15622 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15623 success = (call != nullptr);
15624 }
15625 if (!success) {
15626 ciEnv::current()->record_failure("CodeCache is full");
15627 return;
15628 }
15629 %}
15630
15631 ins_pipe(pipe_class_memory);
15632 %}
15633
15634 // Intrisics for String.compareTo()
15635
15636 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15637 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15638 %{
15639 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15640 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15641 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15642
15643 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15644 ins_encode %{
15645 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15646 __ string_compare($str1$$Register, $str2$$Register,
15647 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15648 $tmp1$$Register, $tmp2$$Register,
15649 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15650 %}
15651 ins_pipe(pipe_class_memory);
15652 %}
15653
15654 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15655 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15656 %{
15657 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15658 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15659 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15660
15661 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15662 ins_encode %{
15663 __ string_compare($str1$$Register, $str2$$Register,
15664 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15665 $tmp1$$Register, $tmp2$$Register,
15666 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15667 %}
15668 ins_pipe(pipe_class_memory);
15669 %}
15670
15671 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15672 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15673 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15674 %{
15675 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15676 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15677 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15678 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15679
15680 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15681 ins_encode %{
15682 __ string_compare($str1$$Register, $str2$$Register,
15683 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15684 $tmp1$$Register, $tmp2$$Register,
15685 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15686 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15687 %}
15688 ins_pipe(pipe_class_memory);
15689 %}
15690
15691 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15692 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15693 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15694 %{
15695 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15696 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15697 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15698 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15699
15700 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15701 ins_encode %{
15702 __ string_compare($str1$$Register, $str2$$Register,
15703 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15704 $tmp1$$Register, $tmp2$$Register,
15705 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15706 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15707 %}
15708 ins_pipe(pipe_class_memory);
15709 %}
15710
15711 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15712 // these string_compare variants as NEON register type for convenience so that the prototype of
15713 // string_compare can be shared with all variants.
15714
15715 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15716 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15717 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15718 pRegGov_P1 pgtmp2, rFlagsReg cr)
15719 %{
15720 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15721 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15722 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15723 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15724
15725 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15726 ins_encode %{
15727 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15728 __ string_compare($str1$$Register, $str2$$Register,
15729 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15730 $tmp1$$Register, $tmp2$$Register,
15731 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15732 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15733 StrIntrinsicNode::LL);
15734 %}
15735 ins_pipe(pipe_class_memory);
15736 %}
15737
15738 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15739 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15740 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15741 pRegGov_P1 pgtmp2, rFlagsReg cr)
15742 %{
15743 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15744 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15745 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15746 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15747
15748 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15749 ins_encode %{
15750 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15751 __ string_compare($str1$$Register, $str2$$Register,
15752 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15753 $tmp1$$Register, $tmp2$$Register,
15754 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15755 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15756 StrIntrinsicNode::LU);
15757 %}
15758 ins_pipe(pipe_class_memory);
15759 %}
15760
15761 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15762 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15763 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15764 pRegGov_P1 pgtmp2, rFlagsReg cr)
15765 %{
15766 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15767 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15768 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15769 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15770
15771 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15772 ins_encode %{
15773 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15774 __ string_compare($str1$$Register, $str2$$Register,
15775 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15776 $tmp1$$Register, $tmp2$$Register,
15777 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15778 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15779 StrIntrinsicNode::UL);
15780 %}
15781 ins_pipe(pipe_class_memory);
15782 %}
15783
15784 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15785 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15786 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15787 pRegGov_P1 pgtmp2, rFlagsReg cr)
15788 %{
15789 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15790 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15791 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15792 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15793
15794 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15795 ins_encode %{
15796 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15797 __ string_compare($str1$$Register, $str2$$Register,
15798 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15799 $tmp1$$Register, $tmp2$$Register,
15800 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15801 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15802 StrIntrinsicNode::UU);
15803 %}
15804 ins_pipe(pipe_class_memory);
15805 %}
15806
15807 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15808 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15809 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15810 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15811 %{
15812 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15813 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15814 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15815 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15816 TEMP vtmp0, TEMP vtmp1, KILL cr);
15817 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15818 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15819
15820 ins_encode %{
15821 __ string_indexof($str1$$Register, $str2$$Register,
15822 $cnt1$$Register, $cnt2$$Register,
15823 $tmp1$$Register, $tmp2$$Register,
15824 $tmp3$$Register, $tmp4$$Register,
15825 $tmp5$$Register, $tmp6$$Register,
15826 -1, $result$$Register, StrIntrinsicNode::UU);
15827 %}
15828 ins_pipe(pipe_class_memory);
15829 %}
15830
15831 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15832 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15833 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15834 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15835 %{
15836 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15837 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15838 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15839 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15840 TEMP vtmp0, TEMP vtmp1, KILL cr);
15841 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15842 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15843
15844 ins_encode %{
15845 __ string_indexof($str1$$Register, $str2$$Register,
15846 $cnt1$$Register, $cnt2$$Register,
15847 $tmp1$$Register, $tmp2$$Register,
15848 $tmp3$$Register, $tmp4$$Register,
15849 $tmp5$$Register, $tmp6$$Register,
15850 -1, $result$$Register, StrIntrinsicNode::LL);
15851 %}
15852 ins_pipe(pipe_class_memory);
15853 %}
15854
15855 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15856 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15857 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15858 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15859 %{
15860 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15861 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15862 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15863 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15864 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15865 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15866 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15867
15868 ins_encode %{
15869 __ string_indexof($str1$$Register, $str2$$Register,
15870 $cnt1$$Register, $cnt2$$Register,
15871 $tmp1$$Register, $tmp2$$Register,
15872 $tmp3$$Register, $tmp4$$Register,
15873 $tmp5$$Register, $tmp6$$Register,
15874 -1, $result$$Register, StrIntrinsicNode::UL);
15875 %}
15876 ins_pipe(pipe_class_memory);
15877 %}
15878
15879 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15880 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15881 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15882 %{
15883 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15884 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15885 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15886 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15887 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15888 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15889
15890 ins_encode %{
15891 int icnt2 = (int)$int_cnt2$$constant;
15892 __ string_indexof($str1$$Register, $str2$$Register,
15893 $cnt1$$Register, zr,
15894 $tmp1$$Register, $tmp2$$Register,
15895 $tmp3$$Register, $tmp4$$Register, zr, zr,
15896 icnt2, $result$$Register, StrIntrinsicNode::UU);
15897 %}
15898 ins_pipe(pipe_class_memory);
15899 %}
15900
15901 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15902 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15903 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15904 %{
15905 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15906 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15907 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15908 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15909 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15910 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15911
15912 ins_encode %{
15913 int icnt2 = (int)$int_cnt2$$constant;
15914 __ string_indexof($str1$$Register, $str2$$Register,
15915 $cnt1$$Register, zr,
15916 $tmp1$$Register, $tmp2$$Register,
15917 $tmp3$$Register, $tmp4$$Register, zr, zr,
15918 icnt2, $result$$Register, StrIntrinsicNode::LL);
15919 %}
15920 ins_pipe(pipe_class_memory);
15921 %}
15922
15923 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15924 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15925 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15926 %{
15927 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15928 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15929 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15930 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15931 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15932 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15933
15934 ins_encode %{
15935 int icnt2 = (int)$int_cnt2$$constant;
15936 __ string_indexof($str1$$Register, $str2$$Register,
15937 $cnt1$$Register, zr,
15938 $tmp1$$Register, $tmp2$$Register,
15939 $tmp3$$Register, $tmp4$$Register, zr, zr,
15940 icnt2, $result$$Register, StrIntrinsicNode::UL);
15941 %}
15942 ins_pipe(pipe_class_memory);
15943 %}
15944
15945 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15946 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15947 iRegINoSp tmp3, rFlagsReg cr)
15948 %{
15949 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15950 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15951 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15952 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15953
15954 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15955
15956 ins_encode %{
15957 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15958 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15959 $tmp3$$Register);
15960 %}
15961 ins_pipe(pipe_class_memory);
15962 %}
15963
15964 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15965 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15966 iRegINoSp tmp3, rFlagsReg cr)
15967 %{
15968 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15969 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15970 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15971 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15972
15973 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15974
15975 ins_encode %{
15976 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15977 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15978 $tmp3$$Register);
15979 %}
15980 ins_pipe(pipe_class_memory);
15981 %}
15982
15983 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15984 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15985 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15986 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15987 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15988 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15989 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15990 ins_encode %{
15991 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15992 $result$$Register, $ztmp1$$FloatRegister,
15993 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15994 $ptmp$$PRegister, true /* isL */);
15995 %}
15996 ins_pipe(pipe_class_memory);
15997 %}
15998
15999 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16000 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16001 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16002 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16003 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16004 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16005 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16006 ins_encode %{
16007 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16008 $result$$Register, $ztmp1$$FloatRegister,
16009 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16010 $ptmp$$PRegister, false /* isL */);
16011 %}
16012 ins_pipe(pipe_class_memory);
16013 %}
16014
16015 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16016 iRegI_R0 result, rFlagsReg cr)
16017 %{
16018 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16019 match(Set result (StrEquals (Binary str1 str2) cnt));
16020 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16021
16022 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16023 ins_encode %{
16024 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16025 __ string_equals($str1$$Register, $str2$$Register,
16026 $result$$Register, $cnt$$Register);
16027 %}
16028 ins_pipe(pipe_class_memory);
16029 %}
16030
16031 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16032 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16033 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16034 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16035 iRegP_R10 tmp, rFlagsReg cr)
16036 %{
16037 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16038 match(Set result (AryEq ary1 ary2));
16039 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16040 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16041 TEMP vtmp6, TEMP vtmp7, KILL cr);
16042
16043 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16044 ins_encode %{
16045 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16046 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16047 $result$$Register, $tmp$$Register, 1);
16048 if (tpc == nullptr) {
16049 ciEnv::current()->record_failure("CodeCache is full");
16050 return;
16051 }
16052 %}
16053 ins_pipe(pipe_class_memory);
16054 %}
16055
16056 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16057 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16058 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16059 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16060 iRegP_R10 tmp, rFlagsReg cr)
16061 %{
16062 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16063 match(Set result (AryEq ary1 ary2));
16064 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16065 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16066 TEMP vtmp6, TEMP vtmp7, KILL cr);
16067
16068 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16069 ins_encode %{
16070 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16071 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16072 $result$$Register, $tmp$$Register, 2);
16073 if (tpc == nullptr) {
16074 ciEnv::current()->record_failure("CodeCache is full");
16075 return;
16076 }
16077 %}
16078 ins_pipe(pipe_class_memory);
16079 %}
16080
16081 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16082 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16083 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16084 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16085 %{
16086 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16087 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16088 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16089
16090 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16091 ins_encode %{
16092 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16093 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16094 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16095 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16096 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16097 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16098 (BasicType)$basic_type$$constant);
16099 if (tpc == nullptr) {
16100 ciEnv::current()->record_failure("CodeCache is full");
16101 return;
16102 }
16103 %}
16104 ins_pipe(pipe_class_memory);
16105 %}
16106
16107 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16108 %{
16109 match(Set result (CountPositives ary1 len));
16110 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16111 format %{ "count positives byte[] $ary1,$len -> $result" %}
16112 ins_encode %{
16113 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16114 if (tpc == nullptr) {
16115 ciEnv::current()->record_failure("CodeCache is full");
16116 return;
16117 }
16118 %}
16119 ins_pipe( pipe_slow );
16120 %}
16121
16122 // fast char[] to byte[] compression
16123 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16124 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16125 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16126 iRegI_R0 result, rFlagsReg cr)
16127 %{
16128 match(Set result (StrCompressedCopy src (Binary dst len)));
16129 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16130 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16131
16132 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16133 ins_encode %{
16134 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16135 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16136 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16137 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16138 %}
16139 ins_pipe(pipe_slow);
16140 %}
16141
16142 // fast byte[] to char[] inflation
16143 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16144 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16145 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16146 %{
16147 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16148 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16149 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16150 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16151
16152 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16153 ins_encode %{
16154 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16155 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16156 $vtmp2$$FloatRegister, $tmp$$Register);
16157 if (tpc == nullptr) {
16158 ciEnv::current()->record_failure("CodeCache is full");
16159 return;
16160 }
16161 %}
16162 ins_pipe(pipe_class_memory);
16163 %}
16164
16165 // encode char[] to byte[] in ISO_8859_1
16166 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16167 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16168 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16169 iRegI_R0 result, rFlagsReg cr)
16170 %{
16171 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16172 match(Set result (EncodeISOArray src (Binary dst len)));
16173 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16174 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16175
16176 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16177 ins_encode %{
16178 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16179 $result$$Register, false,
16180 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16181 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16182 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16183 %}
16184 ins_pipe(pipe_class_memory);
16185 %}
16186
16187 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16188 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16189 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16190 iRegI_R0 result, rFlagsReg cr)
16191 %{
16192 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16193 match(Set result (EncodeISOArray src (Binary dst len)));
16194 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16195 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16196
16197 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16198 ins_encode %{
16199 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16200 $result$$Register, true,
16201 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16202 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16203 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16204 %}
16205 ins_pipe(pipe_class_memory);
16206 %}
16207
16208 //----------------------------- CompressBits/ExpandBits ------------------------
16209
16210 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16211 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16212 match(Set dst (CompressBits src mask));
16213 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16214 format %{ "mov $tsrc, $src\n\t"
16215 "mov $tmask, $mask\n\t"
16216 "bext $tdst, $tsrc, $tmask\n\t"
16217 "mov $dst, $tdst"
16218 %}
16219 ins_encode %{
16220 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16221 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16222 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16223 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16224 %}
16225 ins_pipe(pipe_slow);
16226 %}
16227
16228 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16229 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16230 match(Set dst (CompressBits (LoadI mem) mask));
16231 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16232 format %{ "ldrs $tsrc, $mem\n\t"
16233 "ldrs $tmask, $mask\n\t"
16234 "bext $tdst, $tsrc, $tmask\n\t"
16235 "mov $dst, $tdst"
16236 %}
16237 ins_encode %{
16238 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16239 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16240 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16241 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16242 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16243 %}
16244 ins_pipe(pipe_slow);
16245 %}
16246
16247 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16248 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16249 match(Set dst (CompressBits src mask));
16250 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16251 format %{ "mov $tsrc, $src\n\t"
16252 "mov $tmask, $mask\n\t"
16253 "bext $tdst, $tsrc, $tmask\n\t"
16254 "mov $dst, $tdst"
16255 %}
16256 ins_encode %{
16257 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16258 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16259 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16260 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16261 %}
16262 ins_pipe(pipe_slow);
16263 %}
16264
16265 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16266 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16267 match(Set dst (CompressBits (LoadL mem) mask));
16268 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16269 format %{ "ldrd $tsrc, $mem\n\t"
16270 "ldrd $tmask, $mask\n\t"
16271 "bext $tdst, $tsrc, $tmask\n\t"
16272 "mov $dst, $tdst"
16273 %}
16274 ins_encode %{
16275 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16276 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16277 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16278 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16279 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16280 %}
16281 ins_pipe(pipe_slow);
16282 %}
16283
16284 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16285 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16286 match(Set dst (ExpandBits src mask));
16287 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16288 format %{ "mov $tsrc, $src\n\t"
16289 "mov $tmask, $mask\n\t"
16290 "bdep $tdst, $tsrc, $tmask\n\t"
16291 "mov $dst, $tdst"
16292 %}
16293 ins_encode %{
16294 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16295 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16296 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16297 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16298 %}
16299 ins_pipe(pipe_slow);
16300 %}
16301
16302 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16303 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16304 match(Set dst (ExpandBits (LoadI mem) mask));
16305 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16306 format %{ "ldrs $tsrc, $mem\n\t"
16307 "ldrs $tmask, $mask\n\t"
16308 "bdep $tdst, $tsrc, $tmask\n\t"
16309 "mov $dst, $tdst"
16310 %}
16311 ins_encode %{
16312 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16313 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16314 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16315 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16316 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16317 %}
16318 ins_pipe(pipe_slow);
16319 %}
16320
16321 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16322 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16323 match(Set dst (ExpandBits src mask));
16324 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16325 format %{ "mov $tsrc, $src\n\t"
16326 "mov $tmask, $mask\n\t"
16327 "bdep $tdst, $tsrc, $tmask\n\t"
16328 "mov $dst, $tdst"
16329 %}
16330 ins_encode %{
16331 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16332 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16333 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16334 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16335 %}
16336 ins_pipe(pipe_slow);
16337 %}
16338
16339
16340 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16341 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16342 match(Set dst (ExpandBits (LoadL mem) mask));
16343 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16344 format %{ "ldrd $tsrc, $mem\n\t"
16345 "ldrd $tmask, $mask\n\t"
16346 "bdep $tdst, $tsrc, $tmask\n\t"
16347 "mov $dst, $tdst"
16348 %}
16349 ins_encode %{
16350 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16351 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16352 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16353 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16354 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16355 %}
16356 ins_pipe(pipe_slow);
16357 %}
16358
16359 //----------------------------- Reinterpret ----------------------------------
16360 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16361 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16362 match(Set dst (ReinterpretHF2S src));
16363 format %{ "reinterpretHF2S $dst, $src" %}
16364 ins_encode %{
16365 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16366 %}
16367 ins_pipe(pipe_slow);
16368 %}
16369
16370 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16371 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16372 match(Set dst (ReinterpretS2HF src));
16373 format %{ "reinterpretS2HF $dst, $src" %}
16374 ins_encode %{
16375 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16376 %}
16377 ins_pipe(pipe_slow);
16378 %}
16379
16380 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16381 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16382 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16383 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16384 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16385 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16386 // can be omitted in this pattern, resulting in -
16387 // fcvt $dst, $src // Convert float to half-precision float
16388 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16389 %{
16390 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16391 format %{ "convF2HFAndS2HF $dst, $src" %}
16392 ins_encode %{
16393 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16394 %}
16395 ins_pipe(pipe_slow);
16396 %}
16397
16398 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16399 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16400 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16401 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16402 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16403 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16404 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16405 // resulting in -
16406 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16407 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16408 %{
16409 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16410 format %{ "convHF2SAndHF2F $dst, $src" %}
16411 ins_encode %{
16412 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16413 %}
16414 ins_pipe(pipe_slow);
16415 %}
16416
16417 // ============================================================================
16418 // This name is KNOWN by the ADLC and cannot be changed.
16419 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16420 // for this guy.
16421 instruct tlsLoadP(thread_RegP dst)
16422 %{
16423 match(Set dst (ThreadLocal));
16424
16425 ins_cost(0);
16426
16427 format %{ " -- \t// $dst=Thread::current(), empty" %}
16428
16429 size(0);
16430
16431 ins_encode( /*empty*/ );
16432
16433 ins_pipe(pipe_class_empty);
16434 %}
16435
16436 //----------PEEPHOLE RULES-----------------------------------------------------
16437 // These must follow all instruction definitions as they use the names
16438 // defined in the instructions definitions.
16439 //
16440 // peepmatch ( root_instr_name [preceding_instruction]* );
16441 //
16442 // peepconstraint %{
16443 // (instruction_number.operand_name relational_op instruction_number.operand_name
16444 // [, ...] );
16445 // // instruction numbers are zero-based using left to right order in peepmatch
16446 //
16447 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16448 // // provide an instruction_number.operand_name for each operand that appears
16449 // // in the replacement instruction's match rule
16450 //
16451 // ---------VM FLAGS---------------------------------------------------------
16452 //
16453 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16454 //
16455 // Each peephole rule is given an identifying number starting with zero and
16456 // increasing by one in the order seen by the parser. An individual peephole
16457 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16458 // on the command-line.
16459 //
16460 // ---------CURRENT LIMITATIONS----------------------------------------------
16461 //
16462 // Only match adjacent instructions in same basic block
16463 // Only equality constraints
16464 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16465 // Only one replacement instruction
16466 //
16467 // ---------EXAMPLE----------------------------------------------------------
16468 //
16469 // // pertinent parts of existing instructions in architecture description
16470 // instruct movI(iRegINoSp dst, iRegI src)
16471 // %{
16472 // match(Set dst (CopyI src));
16473 // %}
16474 //
16475 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16476 // %{
16477 // match(Set dst (AddI dst src));
16478 // effect(KILL cr);
16479 // %}
16480 //
16481 // // Change (inc mov) to lea
16482 // peephole %{
16483 // // increment preceded by register-register move
16484 // peepmatch ( incI_iReg movI );
16485 // // require that the destination register of the increment
16486 // // match the destination register of the move
16487 // peepconstraint ( 0.dst == 1.dst );
16488 // // construct a replacement instruction that sets
16489 // // the destination to ( move's source register + one )
16490 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16491 // %}
16492 //
16493
16494 // Implementation no longer uses movX instructions since
16495 // machine-independent system no longer uses CopyX nodes.
16496 //
16497 // peephole
16498 // %{
16499 // peepmatch (incI_iReg movI);
16500 // peepconstraint (0.dst == 1.dst);
16501 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16502 // %}
16503
16504 // peephole
16505 // %{
16506 // peepmatch (decI_iReg movI);
16507 // peepconstraint (0.dst == 1.dst);
16508 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16509 // %}
16510
16511 // peephole
16512 // %{
16513 // peepmatch (addI_iReg_imm movI);
16514 // peepconstraint (0.dst == 1.dst);
16515 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16516 // %}
16517
16518 // peephole
16519 // %{
16520 // peepmatch (incL_iReg movL);
16521 // peepconstraint (0.dst == 1.dst);
16522 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16523 // %}
16524
16525 // peephole
16526 // %{
16527 // peepmatch (decL_iReg movL);
16528 // peepconstraint (0.dst == 1.dst);
16529 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16530 // %}
16531
16532 // peephole
16533 // %{
16534 // peepmatch (addL_iReg_imm movL);
16535 // peepconstraint (0.dst == 1.dst);
16536 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16537 // %}
16538
16539 // peephole
16540 // %{
16541 // peepmatch (addP_iReg_imm movP);
16542 // peepconstraint (0.dst == 1.dst);
16543 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16544 // %}
16545
16546 // // Change load of spilled value to only a spill
16547 // instruct storeI(memory mem, iRegI src)
16548 // %{
16549 // match(Set mem (StoreI mem src));
16550 // %}
16551 //
16552 // instruct loadI(iRegINoSp dst, memory mem)
16553 // %{
16554 // match(Set dst (LoadI mem));
16555 // %}
16556 //
16557
16558 //----------SMARTSPILL RULES---------------------------------------------------
16559 // These must follow all instruction definitions as they use the names
16560 // defined in the instructions definitions.
16561
16562 // Local Variables:
16563 // mode: c++
16564 // End: