1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_CompareAndSwapB:
1512 case Op_CompareAndSwapS:
1513 case Op_GetAndSetI:
1514 case Op_GetAndSetL:
1515 case Op_GetAndSetP:
1516 case Op_GetAndSetN:
1517 case Op_GetAndAddI:
1518 case Op_GetAndAddL:
1519 return true;
1520 case Op_CompareAndExchangeI:
1521 case Op_CompareAndExchangeN:
1522 case Op_CompareAndExchangeB:
1523 case Op_CompareAndExchangeS:
1524 case Op_CompareAndExchangeL:
1525 case Op_CompareAndExchangeP:
1526 case Op_WeakCompareAndSwapB:
1527 case Op_WeakCompareAndSwapS:
1528 case Op_WeakCompareAndSwapI:
1529 case Op_WeakCompareAndSwapL:
1530 case Op_WeakCompareAndSwapP:
1531 case Op_WeakCompareAndSwapN:
1532 return maybe_volatile;
1533 default:
1534 return false;
1535 }
1536 }
1537
1538 // helper to determine the maximum number of Phi nodes we may need to
1539 // traverse when searching from a card mark membar for the merge mem
1540 // feeding a trailing membar or vice versa
1541
1542 // predicates controlling emit of ldr<x>/ldar<x>
1543
1544 bool unnecessary_acquire(const Node *barrier)
1545 {
1546 assert(barrier->is_MemBar(), "expecting a membar");
1547
1548 MemBarNode* mb = barrier->as_MemBar();
1549
1550 if (mb->trailing_load()) {
1551 return true;
1552 }
1553
1554 if (mb->trailing_load_store()) {
1555 Node* load_store = mb->in(MemBarNode::Precedent);
1556 assert(load_store->is_LoadStore(), "unexpected graph shape");
1557 return is_CAS(load_store->Opcode(), true);
1558 }
1559
1560 return false;
1561 }
1562
1563 bool needs_acquiring_load(const Node *n)
1564 {
1565 assert(n->is_Load(), "expecting a load");
1566 LoadNode *ld = n->as_Load();
1567 return ld->is_acquire();
1568 }
1569
1570 bool unnecessary_release(const Node *n)
1571 {
1572 assert((n->is_MemBar() &&
1573 n->Opcode() == Op_MemBarRelease),
1574 "expecting a release membar");
1575
1576 MemBarNode *barrier = n->as_MemBar();
1577 if (!barrier->leading()) {
1578 return false;
1579 } else {
1580 Node* trailing = barrier->trailing_membar();
1581 MemBarNode* trailing_mb = trailing->as_MemBar();
1582 assert(trailing_mb->trailing(), "Not a trailing membar?");
1583 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1584
1585 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1586 if (mem->is_Store()) {
1587 assert(mem->as_Store()->is_release(), "");
1588 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1589 return true;
1590 } else {
1591 assert(mem->is_LoadStore(), "");
1592 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1593 return is_CAS(mem->Opcode(), true);
1594 }
1595 }
1596 return false;
1597 }
1598
1599 bool unnecessary_volatile(const Node *n)
1600 {
1601 // assert n->is_MemBar();
1602 MemBarNode *mbvol = n->as_MemBar();
1603
1604 bool release = mbvol->trailing_store();
1605 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1606 #ifdef ASSERT
1607 if (release) {
1608 Node* leading = mbvol->leading_membar();
1609 assert(leading->Opcode() == Op_MemBarRelease, "");
1610 assert(leading->as_MemBar()->leading_store(), "");
1611 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1612 }
1613 #endif
1614
1615 return release;
1616 }
1617
1618 // predicates controlling emit of str<x>/stlr<x>
1619
1620 bool needs_releasing_store(const Node *n)
1621 {
1622 // assert n->is_Store();
1623 StoreNode *st = n->as_Store();
1624 return st->trailing_membar() != nullptr;
1625 }
1626
1627 // predicate controlling translation of CAS
1628 //
1629 // returns true if CAS needs to use an acquiring load otherwise false
1630
1631 bool needs_acquiring_load_exclusive(const Node *n)
1632 {
1633 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1634 LoadStoreNode* ldst = n->as_LoadStore();
1635 if (is_CAS(n->Opcode(), false)) {
1636 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1637 } else {
1638 return ldst->trailing_membar() != nullptr;
1639 }
1640
1641 // so we can just return true here
1642 return true;
1643 }
1644
1645 #define __ masm->
1646
1647 // advance declarations for helper functions to convert register
1648 // indices to register objects
1649
1650 // the ad file has to provide implementations of certain methods
1651 // expected by the generic code
1652 //
1653 // REQUIRED FUNCTIONALITY
1654
1655 //=============================================================================
1656
1657 // !!!!! Special hack to get all types of calls to specify the byte offset
1658 // from the start of the call to the point where the return address
1659 // will point.
1660
1661 int MachCallStaticJavaNode::ret_addr_offset()
1662 {
1663 // call should be a simple bl
1664 int off = 4;
1665 return off;
1666 }
1667
1668 int MachCallDynamicJavaNode::ret_addr_offset()
1669 {
1670 return 16; // movz, movk, movk, bl
1671 }
1672
1673 int MachCallRuntimeNode::ret_addr_offset() {
1674 // for generated stubs the call will be
1675 // bl(addr)
1676 // or with far branches
1677 // bl(trampoline_stub)
1678 // for real runtime callouts it will be six instructions
1679 // see aarch64_enc_java_to_runtime
1680 // adr(rscratch2, retaddr)
1681 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1682 // lea(rscratch1, RuntimeAddress(addr)
1683 // blr(rscratch1)
1684 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1685 if (cb) {
1686 return 1 * NativeInstruction::instruction_size;
1687 } else if (_entry_point == nullptr) {
1688 // See CallLeafNoFPIndirect
1689 return 1 * NativeInstruction::instruction_size;
1690 } else {
1691 return 6 * NativeInstruction::instruction_size;
1692 }
1693 }
1694
1695 //=============================================================================
1696
1697 #ifndef PRODUCT
1698 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1699 st->print("BREAKPOINT");
1700 }
1701 #endif
1702
1703 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1704 __ brk(0);
1705 }
1706
1707 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1708 return MachNode::size(ra_);
1709 }
1710
1711 //=============================================================================
1712
1713 #ifndef PRODUCT
1714 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1715 st->print("nop \t# %d bytes pad for loops and calls", _count);
1716 }
1717 #endif
1718
1719 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1720 for (int i = 0; i < _count; i++) {
1721 __ nop();
1722 }
1723 }
1724
1725 uint MachNopNode::size(PhaseRegAlloc*) const {
1726 return _count * NativeInstruction::instruction_size;
1727 }
1728
1729 //=============================================================================
1730 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1731
1732 int ConstantTable::calculate_table_base_offset() const {
1733 return 0; // absolute addressing, no offset
1734 }
1735
1736 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1737 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1738 ShouldNotReachHere();
1739 }
1740
1741 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1742 // Empty encoding
1743 }
1744
1745 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1746 return 0;
1747 }
1748
1749 #ifndef PRODUCT
1750 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1751 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1752 }
1753 #endif
1754
1755 #ifndef PRODUCT
1756 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1757 Compile* C = ra_->C;
1758
1759 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1760
1761 if (C->output()->need_stack_bang(framesize))
1762 st->print("# stack bang size=%d\n\t", framesize);
1763
1764 if (VM_Version::use_rop_protection()) {
1765 st->print("ldr zr, [lr]\n\t");
1766 st->print("paciaz\n\t");
1767 }
1768 if (framesize < ((1 << 9) + 2 * wordSize)) {
1769 st->print("sub sp, sp, #%d\n\t", framesize);
1770 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1771 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1772 } else {
1773 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1774 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1775 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1776 st->print("sub sp, sp, rscratch1");
1777 }
1778 if (C->stub_function() == nullptr) {
1779 st->print("\n\t");
1780 st->print("ldr rscratch1, [guard]\n\t");
1781 st->print("dmb ishld\n\t");
1782 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1783 st->print("cmp rscratch1, rscratch2\n\t");
1784 st->print("b.eq skip");
1785 st->print("\n\t");
1786 st->print("blr #nmethod_entry_barrier_stub\n\t");
1787 st->print("b skip\n\t");
1788 st->print("guard: int\n\t");
1789 st->print("\n\t");
1790 st->print("skip:\n\t");
1791 }
1792 }
1793 #endif
1794
1795 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1796 Compile* C = ra_->C;
1797
1798
1799 __ verified_entry(C, 0);
1800
1801 if (C->stub_function() == nullptr) {
1802 __ entry_barrier();
1803 }
1804
1805 if (!Compile::current()->output()->in_scratch_emit_size()) {
1806 __ bind(*_verified_entry);
1807 }
1808
1809 if (VerifyStackAtCalls) {
1810 Unimplemented();
1811 }
1812
1813 C->output()->set_frame_complete(__ offset());
1814
1815 if (C->has_mach_constant_base_node()) {
1816 // NOTE: We set the table base offset here because users might be
1817 // emitted before MachConstantBaseNode.
1818 ConstantTable& constant_table = C->output()->constant_table();
1819 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1820 }
1821 }
1822
1823 int MachPrologNode::reloc() const
1824 {
1825 return 0;
1826 }
1827
1828 //=============================================================================
1829
1830 #ifndef PRODUCT
1831 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1832 Compile* C = ra_->C;
1833 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1834
1835 st->print("# pop frame %d\n\t",framesize);
1836
1837 if (framesize == 0) {
1838 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1839 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1840 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1841 st->print("add sp, sp, #%d\n\t", framesize);
1842 } else {
1843 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1844 st->print("add sp, sp, rscratch1\n\t");
1845 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1846 }
1847 if (VM_Version::use_rop_protection()) {
1848 st->print("autiaz\n\t");
1849 st->print("ldr zr, [lr]\n\t");
1850 }
1851
1852 if (do_polling() && C->is_method_compilation()) {
1853 st->print("# test polling word\n\t");
1854 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1855 st->print("cmp sp, rscratch1\n\t");
1856 st->print("bhi #slow_path");
1857 }
1858 }
1859 #endif
1860
1861 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1862 Compile* C = ra_->C;
1863 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1864
1865 __ remove_frame(framesize, C->needs_stack_repair());
1866
1867 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1868 __ reserved_stack_check();
1869 }
1870
1871 if (do_polling() && C->is_method_compilation()) {
1872 Label dummy_label;
1873 Label* code_stub = &dummy_label;
1874 if (!C->output()->in_scratch_emit_size()) {
1875 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1876 C->output()->add_stub(stub);
1877 code_stub = &stub->entry();
1878 }
1879 __ relocate(relocInfo::poll_return_type);
1880 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1881 }
1882 }
1883
1884 int MachEpilogNode::reloc() const {
1885 // Return number of relocatable values contained in this instruction.
1886 return 1; // 1 for polling page.
1887 }
1888
1889 const Pipeline * MachEpilogNode::pipeline() const {
1890 return MachNode::pipeline_class();
1891 }
1892
1893 //=============================================================================
1894
1895 static enum RC rc_class(OptoReg::Name reg) {
1896
1897 if (reg == OptoReg::Bad) {
1898 return rc_bad;
1899 }
1900
1901 // we have 32 int registers * 2 halves
1902 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1903
1904 if (reg < slots_of_int_registers) {
1905 return rc_int;
1906 }
1907
1908 // we have 32 float register * 8 halves
1909 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1910 if (reg < slots_of_int_registers + slots_of_float_registers) {
1911 return rc_float;
1912 }
1913
1914 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1915 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1916 return rc_predicate;
1917 }
1918
1919 // Between predicate regs & stack is the flags.
1920 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1921
1922 return rc_stack;
1923 }
1924
1925 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1926 Compile* C = ra_->C;
1927
1928 // Get registers to move.
1929 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1930 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1931 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1932 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1933
1934 enum RC src_hi_rc = rc_class(src_hi);
1935 enum RC src_lo_rc = rc_class(src_lo);
1936 enum RC dst_hi_rc = rc_class(dst_hi);
1937 enum RC dst_lo_rc = rc_class(dst_lo);
1938
1939 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1940
1941 if (src_hi != OptoReg::Bad && !bottom_type()->isa_pvectmask()) {
1942 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1943 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1944 "expected aligned-adjacent pairs");
1945 }
1946
1947 if (src_lo == dst_lo && src_hi == dst_hi) {
1948 return 0; // Self copy, no move.
1949 }
1950
1951 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1952 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1953 int src_offset = ra_->reg2offset(src_lo);
1954 int dst_offset = ra_->reg2offset(dst_lo);
1955
1956 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
1957 uint ireg = ideal_reg();
1958 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
1959 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
1960 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
1961 if (ireg == Op_VecA && masm) {
1962 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1963 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1964 // stack->stack
1965 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1966 sve_vector_reg_size_in_bytes);
1967 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1968 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1969 sve_vector_reg_size_in_bytes);
1970 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1971 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1972 sve_vector_reg_size_in_bytes);
1973 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1974 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1975 as_FloatRegister(Matcher::_regEncode[src_lo]),
1976 as_FloatRegister(Matcher::_regEncode[src_lo]));
1977 } else {
1978 ShouldNotReachHere();
1979 }
1980 } else if (masm) {
1981 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1982 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1983 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1984 // stack->stack
1985 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1986 if (ireg == Op_VecD) {
1987 __ unspill(rscratch1, true, src_offset);
1988 __ spill(rscratch1, true, dst_offset);
1989 } else {
1990 __ spill_copy128(src_offset, dst_offset);
1991 }
1992 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1993 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1994 ireg == Op_VecD ? __ T8B : __ T16B,
1995 as_FloatRegister(Matcher::_regEncode[src_lo]));
1996 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1997 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1998 ireg == Op_VecD ? __ D : __ Q,
1999 ra_->reg2offset(dst_lo));
2000 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2001 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2002 ireg == Op_VecD ? __ D : __ Q,
2003 ra_->reg2offset(src_lo));
2004 } else {
2005 ShouldNotReachHere();
2006 }
2007 }
2008 } else if (masm) {
2009 switch (src_lo_rc) {
2010 case rc_int:
2011 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2012 if (is64) {
2013 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2014 as_Register(Matcher::_regEncode[src_lo]));
2015 } else {
2016 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2017 as_Register(Matcher::_regEncode[src_lo]));
2018 }
2019 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2020 if (is64) {
2021 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2022 as_Register(Matcher::_regEncode[src_lo]));
2023 } else {
2024 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2025 as_Register(Matcher::_regEncode[src_lo]));
2026 }
2027 } else { // gpr --> stack spill
2028 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2029 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2030 }
2031 break;
2032 case rc_float:
2033 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2034 if (is64) {
2035 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2036 as_FloatRegister(Matcher::_regEncode[src_lo]));
2037 } else {
2038 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2039 as_FloatRegister(Matcher::_regEncode[src_lo]));
2040 }
2041 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2042 if (is64) {
2043 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2044 as_FloatRegister(Matcher::_regEncode[src_lo]));
2045 } else {
2046 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2047 as_FloatRegister(Matcher::_regEncode[src_lo]));
2048 }
2049 } else { // fpr --> stack spill
2050 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2051 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2052 is64 ? __ D : __ S, dst_offset);
2053 }
2054 break;
2055 case rc_stack:
2056 if (dst_lo_rc == rc_int) { // stack --> gpr load
2057 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2058 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2059 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2060 is64 ? __ D : __ S, src_offset);
2061 } else if (dst_lo_rc == rc_predicate) {
2062 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2063 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2064 } else { // stack --> stack copy
2065 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2066 if (ideal_reg() == Op_RegVectMask) {
2067 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2068 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2069 } else {
2070 __ unspill(rscratch1, is64, src_offset);
2071 __ spill(rscratch1, is64, dst_offset);
2072 }
2073 }
2074 break;
2075 case rc_predicate:
2076 if (dst_lo_rc == rc_predicate) {
2077 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2078 } else if (dst_lo_rc == rc_stack) {
2079 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2080 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2081 } else {
2082 assert(false, "bad src and dst rc_class combination.");
2083 ShouldNotReachHere();
2084 }
2085 break;
2086 default:
2087 assert(false, "bad rc_class for spill");
2088 ShouldNotReachHere();
2089 }
2090 }
2091
2092 if (st) {
2093 st->print("spill ");
2094 if (src_lo_rc == rc_stack) {
2095 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2096 } else {
2097 st->print("%s -> ", Matcher::regName[src_lo]);
2098 }
2099 if (dst_lo_rc == rc_stack) {
2100 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2101 } else {
2102 st->print("%s", Matcher::regName[dst_lo]);
2103 }
2104 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2105 int vsize = 0;
2106 switch (ideal_reg()) {
2107 case Op_VecD:
2108 vsize = 64;
2109 break;
2110 case Op_VecX:
2111 vsize = 128;
2112 break;
2113 case Op_VecA:
2114 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2115 break;
2116 default:
2117 assert(false, "bad register type for spill");
2118 ShouldNotReachHere();
2119 }
2120 st->print("\t# vector spill size = %d", vsize);
2121 } else if (ideal_reg() == Op_RegVectMask) {
2122 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2123 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2124 st->print("\t# predicate spill size = %d", vsize);
2125 } else {
2126 st->print("\t# spill size = %d", is64 ? 64 : 32);
2127 }
2128 }
2129
2130 return 0;
2131
2132 }
2133
2134 #ifndef PRODUCT
2135 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2136 if (!ra_)
2137 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2138 else
2139 implementation(nullptr, ra_, false, st);
2140 }
2141 #endif
2142
2143 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2144 implementation(masm, ra_, false, nullptr);
2145 }
2146
2147 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2148 return MachNode::size(ra_);
2149 }
2150
2151 //=============================================================================
2152
2153 #ifndef PRODUCT
2154 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2155 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2156 int reg = ra_->get_reg_first(this);
2157 st->print("add %s, rsp, #%d]\t# box lock",
2158 Matcher::regName[reg], offset);
2159 }
2160 #endif
2161
2162 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2163 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2164 int reg = ra_->get_encode(this);
2165
2166 // This add will handle any 24-bit signed offset. 24 bits allows an
2167 // 8 megabyte stack frame.
2168 __ add(as_Register(reg), sp, offset);
2169 }
2170
2171 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2172 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2173 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2174
2175 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2176 return NativeInstruction::instruction_size;
2177 } else {
2178 return 2 * NativeInstruction::instruction_size;
2179 }
2180 }
2181
2182 ///=============================================================================
2183 #ifndef PRODUCT
2184 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2185 {
2186 st->print_cr("# MachVEPNode");
2187 if (!_verified) {
2188 st->print_cr("\t load_class");
2189 } else {
2190 st->print_cr("\t unpack_inline_arg");
2191 }
2192 }
2193 #endif
2194
2195 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2196 {
2197 if (!_verified) {
2198 __ ic_check(1);
2199 } else {
2200 if (ra_->C->stub_function() == nullptr) {
2201 // Emit the entry barrier in a temporary frame before unpacking because
2202 // it can deopt, which would require packing the scalarized args again.
2203 __ verified_entry(ra_->C, 0);
2204 __ entry_barrier();
2205 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2206 __ remove_frame(framesize, false);
2207 }
2208 // Unpack inline type args passed as oop and then jump to
2209 // the verified entry point (skipping the unverified entry).
2210 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2211 // Emit code for verified entry and save increment for stack repair on return
2212 __ verified_entry(ra_->C, sp_inc);
2213 if (Compile::current()->output()->in_scratch_emit_size()) {
2214 Label dummy_verified_entry;
2215 __ b(dummy_verified_entry);
2216 } else {
2217 __ b(*_verified_entry);
2218 }
2219 }
2220 }
2221
2222 //=============================================================================
2223 #ifndef PRODUCT
2224 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2225 {
2226 st->print_cr("# MachUEPNode");
2227 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2228 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2229 st->print_cr("\tcmpw rscratch1, r10");
2230 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2231 }
2232 #endif
2233
2234 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2235 {
2236 __ ic_check(InteriorEntryAlignment);
2237 }
2238
2239 // REQUIRED EMIT CODE
2240
2241 //=============================================================================
2242
2243 // Emit deopt handler code.
2244 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2245 {
2246 // Note that the code buffer's insts_mark is always relative to insts.
2247 // That's why we must use the macroassembler to generate a handler.
2248 address base = __ start_a_stub(size_deopt_handler());
2249 if (base == nullptr) {
2250 ciEnv::current()->record_failure("CodeCache is full");
2251 return 0; // CodeBuffer::expand failed
2252 }
2253
2254 int offset = __ offset();
2255 Label start;
2256 __ bind(start);
2257 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2258
2259 int entry_offset = __ offset();
2260 __ b(start);
2261
2262 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2263 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2264 "out of bounds read in post-call NOP check");
2265 __ end_a_stub();
2266 return entry_offset;
2267 }
2268
2269 // REQUIRED MATCHER CODE
2270
2271 //=============================================================================
2272
2273 bool Matcher::match_rule_supported(int opcode) {
2274 if (!has_match_rule(opcode))
2275 return false;
2276
2277 switch (opcode) {
2278 case Op_OnSpinWait:
2279 return VM_Version::supports_on_spin_wait();
2280 case Op_CacheWB:
2281 case Op_CacheWBPreSync:
2282 case Op_CacheWBPostSync:
2283 if (!VM_Version::supports_data_cache_line_flush()) {
2284 return false;
2285 }
2286 break;
2287 case Op_ExpandBits:
2288 case Op_CompressBits:
2289 if (!VM_Version::supports_svebitperm()) {
2290 return false;
2291 }
2292 break;
2293 case Op_FmaF:
2294 case Op_FmaD:
2295 case Op_FmaVF:
2296 case Op_FmaVD:
2297 if (!UseFMA) {
2298 return false;
2299 }
2300 break;
2301 case Op_FmaHF:
2302 // UseFMA flag also needs to be checked along with FEAT_FP16
2303 if (!UseFMA || !is_feat_fp16_supported()) {
2304 return false;
2305 }
2306 break;
2307 case Op_AddHF:
2308 case Op_SubHF:
2309 case Op_MulHF:
2310 case Op_DivHF:
2311 case Op_MinHF:
2312 case Op_MaxHF:
2313 case Op_SqrtHF:
2314 // Half-precision floating point scalar operations require FEAT_FP16
2315 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2316 // features are supported.
2317 if (!is_feat_fp16_supported()) {
2318 return false;
2319 }
2320 break;
2321 }
2322
2323 return true; // Per default match rules are supported.
2324 }
2325
2326 const RegMask* Matcher::predicate_reg_mask(void) {
2327 return &_PR_REG_mask;
2328 }
2329
2330 bool Matcher::supports_vector_calling_convention(void) {
2331 return EnableVectorSupport;
2332 }
2333
2334 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2335 assert(EnableVectorSupport, "sanity");
2336 int lo = V0_num;
2337 int hi = V0_H_num;
2338 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2339 hi = V0_K_num;
2340 }
2341 return OptoRegPair(hi, lo);
2342 }
2343
2344 // Is this branch offset short enough that a short branch can be used?
2345 //
2346 // NOTE: If the platform does not provide any short branch variants, then
2347 // this method should return false for offset 0.
2348 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2349 // The passed offset is relative to address of the branch.
2350
2351 return (-32768 <= offset && offset < 32768);
2352 }
2353
2354 // Vector width in bytes.
2355 int Matcher::vector_width_in_bytes(BasicType bt) {
2356 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2357 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2358 // Minimum 2 values in vector
2359 if (size < 2*type2aelembytes(bt)) size = 0;
2360 // But never < 4
2361 if (size < 4) size = 0;
2362 return size;
2363 }
2364
2365 // Limits on vector size (number of elements) loaded into vector.
2366 int Matcher::max_vector_size(const BasicType bt) {
2367 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2368 }
2369
2370 int Matcher::min_vector_size(const BasicType bt) {
2371 // Usually, the shortest vector length supported by AArch64 ISA and
2372 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2373 // vectors in a few special cases.
2374 int size;
2375 switch(bt) {
2376 case T_BOOLEAN:
2377 // Load/store a vector mask with only 2 elements for vector types
2378 // such as "2I/2F/2L/2D".
2379 size = 2;
2380 break;
2381 case T_BYTE:
2382 // Generate a "4B" vector, to support vector cast between "8B/16B"
2383 // and "4S/4I/4L/4F/4D".
2384 size = 4;
2385 break;
2386 case T_SHORT:
2387 // Generate a "2S" vector, to support vector cast between "4S/8S"
2388 // and "2I/2L/2F/2D".
2389 size = 2;
2390 break;
2391 default:
2392 // Limit the min vector length to 64-bit.
2393 size = 8 / type2aelembytes(bt);
2394 // The number of elements in a vector should be at least 2.
2395 size = MAX2(size, 2);
2396 }
2397
2398 int max_size = max_vector_size(bt);
2399 return MIN2(size, max_size);
2400 }
2401
2402 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2403 return Matcher::max_vector_size(bt);
2404 }
2405
2406 // Actual max scalable vector register length.
2407 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2408 return Matcher::max_vector_size(bt);
2409 }
2410
2411 // Vector ideal reg.
2412 uint Matcher::vector_ideal_reg(int len) {
2413 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2414 return Op_VecA;
2415 }
2416 switch(len) {
2417 // For 16-bit/32-bit mask vector, reuse VecD.
2418 case 2:
2419 case 4:
2420 case 8: return Op_VecD;
2421 case 16: return Op_VecX;
2422 }
2423 ShouldNotReachHere();
2424 return 0;
2425 }
2426
2427 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2428 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2429 switch (ideal_reg) {
2430 case Op_VecA: return new vecAOper();
2431 case Op_VecD: return new vecDOper();
2432 case Op_VecX: return new vecXOper();
2433 }
2434 ShouldNotReachHere();
2435 return nullptr;
2436 }
2437
2438 bool Matcher::is_reg2reg_move(MachNode* m) {
2439 return false;
2440 }
2441
2442 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2443 return false;
2444 }
2445
2446 bool Matcher::is_generic_vector(MachOper* opnd) {
2447 return opnd->opcode() == VREG;
2448 }
2449
2450 #ifdef ASSERT
2451 // Return whether or not this register is ever used as an argument.
2452 bool Matcher::can_be_java_arg(int reg)
2453 {
2454 return
2455 reg == R0_num || reg == R0_H_num ||
2456 reg == R1_num || reg == R1_H_num ||
2457 reg == R2_num || reg == R2_H_num ||
2458 reg == R3_num || reg == R3_H_num ||
2459 reg == R4_num || reg == R4_H_num ||
2460 reg == R5_num || reg == R5_H_num ||
2461 reg == R6_num || reg == R6_H_num ||
2462 reg == R7_num || reg == R7_H_num ||
2463 reg == V0_num || reg == V0_H_num ||
2464 reg == V1_num || reg == V1_H_num ||
2465 reg == V2_num || reg == V2_H_num ||
2466 reg == V3_num || reg == V3_H_num ||
2467 reg == V4_num || reg == V4_H_num ||
2468 reg == V5_num || reg == V5_H_num ||
2469 reg == V6_num || reg == V6_H_num ||
2470 reg == V7_num || reg == V7_H_num;
2471 }
2472 #endif
2473
2474 uint Matcher::int_pressure_limit()
2475 {
2476 // JDK-8183543: When taking the number of available registers as int
2477 // register pressure threshold, the jtreg test:
2478 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2479 // failed due to C2 compilation failure with
2480 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2481 //
2482 // A derived pointer is live at CallNode and then is flagged by RA
2483 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2484 // derived pointers and lastly fail to spill after reaching maximum
2485 // number of iterations. Lowering the default pressure threshold to
2486 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2487 // a high register pressure area of the code so that split_DEF can
2488 // generate DefinitionSpillCopy for the derived pointer.
2489 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2490 if (!PreserveFramePointer) {
2491 // When PreserveFramePointer is off, frame pointer is allocatable,
2492 // but different from other SOC registers, it is excluded from
2493 // fatproj's mask because its save type is No-Save. Decrease 1 to
2494 // ensure high pressure at fatproj when PreserveFramePointer is off.
2495 // See check_pressure_at_fatproj().
2496 default_int_pressure_threshold--;
2497 }
2498 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2499 }
2500
2501 uint Matcher::float_pressure_limit()
2502 {
2503 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2504 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2505 }
2506
2507 const RegMask& Matcher::firstI_proj_mask() {
2508 ShouldNotReachHere();
2509 return RegMask::EMPTY;
2510 }
2511
2512 // Register for the second projection of an int pair
2513 const RegMask& Matcher::secondI_proj_mask() {
2514 ShouldNotReachHere();
2515 return RegMask::EMPTY;
2516 }
2517
2518 // Register for the first projection of a long pair
2519 const RegMask& Matcher::firstL_proj_mask() {
2520 ShouldNotReachHere();
2521 return RegMask::EMPTY;
2522 }
2523
2524 // Register for the second projection of a long pair
2525 const RegMask& Matcher::secondL_proj_mask() {
2526 ShouldNotReachHere();
2527 return RegMask::EMPTY;
2528 }
2529
2530 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2531 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2532 Node* u = addp->fast_out(i);
2533 if (u->is_LoadStore()) {
2534 // On AArch64, LoadStoreNodes (i.e. compare and swap
2535 // instructions) only take register indirect as an operand, so
2536 // any attempt to use an AddPNode as an input to a LoadStoreNode
2537 // must fail.
2538 return false;
2539 }
2540 if (u->is_Mem()) {
2541 int opsize = u->as_Mem()->memory_size();
2542 assert(opsize > 0, "unexpected memory operand size");
2543 if (u->as_Mem()->memory_size() != (1<<shift)) {
2544 return false;
2545 }
2546 }
2547 }
2548 return true;
2549 }
2550
2551 // Convert BoolTest condition to Assembler condition.
2552 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2553 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2554 Assembler::Condition result;
2555 switch(cond) {
2556 case BoolTest::eq:
2557 result = Assembler::EQ; break;
2558 case BoolTest::ne:
2559 result = Assembler::NE; break;
2560 case BoolTest::le:
2561 result = Assembler::LE; break;
2562 case BoolTest::ge:
2563 result = Assembler::GE; break;
2564 case BoolTest::lt:
2565 result = Assembler::LT; break;
2566 case BoolTest::gt:
2567 result = Assembler::GT; break;
2568 case BoolTest::ule:
2569 result = Assembler::LS; break;
2570 case BoolTest::uge:
2571 result = Assembler::HS; break;
2572 case BoolTest::ult:
2573 result = Assembler::LO; break;
2574 case BoolTest::ugt:
2575 result = Assembler::HI; break;
2576 case BoolTest::overflow:
2577 result = Assembler::VS; break;
2578 case BoolTest::no_overflow:
2579 result = Assembler::VC; break;
2580 default:
2581 ShouldNotReachHere();
2582 return Assembler::Condition(-1);
2583 }
2584
2585 // Check conversion
2586 if (cond & BoolTest::unsigned_compare) {
2587 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2588 } else {
2589 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2590 }
2591
2592 return result;
2593 }
2594
2595 // Binary src (Replicate con)
2596 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2597 if (n == nullptr || m == nullptr) {
2598 return false;
2599 }
2600
2601 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2602 return false;
2603 }
2604
2605 Node* imm_node = m->in(1);
2606 if (!imm_node->is_Con()) {
2607 return false;
2608 }
2609
2610 const Type* t = imm_node->bottom_type();
2611 if (!(t->isa_int() || t->isa_long())) {
2612 return false;
2613 }
2614
2615 switch (n->Opcode()) {
2616 case Op_AndV:
2617 case Op_OrV:
2618 case Op_XorV: {
2619 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2620 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2621 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2622 }
2623 case Op_AddVB:
2624 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2625 case Op_AddVS:
2626 case Op_AddVI:
2627 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2628 case Op_AddVL:
2629 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2630 default:
2631 return false;
2632 }
2633 }
2634
2635 // (XorV src (Replicate m1))
2636 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2637 if (n != nullptr && m != nullptr) {
2638 return n->Opcode() == Op_XorV &&
2639 VectorNode::is_all_ones_vector(m);
2640 }
2641 return false;
2642 }
2643
2644 // Returns true if (n, m) matches "(XorVMask vm2 (MaskAll m1))" and that XorVMask
2645 // is used only by an AndVMask. In that case, cloning m (the MaskAll) lets the
2646 // matcher avoid sharing the MaskAll node and subsume the pattern into rule:
2647 // "(AndVMask vm1 (XorVMask vm2 (MaskAll m1)))".
2648 //
2649 // Limitation: the "andNot" rule still cannot be matched if "m" has other
2650 // uses outside this pattern.
2651 static bool is_vector_mask_not_operand_in_andnot_pattern(Node* n, Node* m) {
2652 if (n == nullptr || m == nullptr) {
2653 return false;
2654 }
2655
2656 if (VectorNode::is_all_ones_vector(m) &&
2657 n->Opcode() == Op_XorVMask &&
2658 n->outcnt() == 1 &&
2659 n->unique_out()->Opcode() == Op_AndVMask) {
2660 // If another input of the AndVMask is also a mask-not pattern that would
2661 // qualify for the `maskAll` cloning, do not clone the "maskAll" here,
2662 // because the match rule can only consume one such pattern.
2663 Node* use = n->unique_out();
2664 Node* other_input = use->in(1) == n ? use->in(2) : use->in(1);
2665 return !VectorNode::is_vectormask_bitwise_not_pattern(other_input);
2666 }
2667 return false;
2668 }
2669
2670 // Should the matcher clone input 'm' of node 'n'?
2671 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2672 if (is_vshift_con_pattern(n, m) ||
2673 is_vector_bitwise_not_pattern(n, m) ||
2674 is_vector_mask_not_operand_in_andnot_pattern(n, m) ||
2675 is_valid_sve_arith_imm_pattern(n, m) ||
2676 is_encode_and_store_pattern(n, m)) {
2677 mstack.push(m, Visit);
2678 return true;
2679 }
2680 return false;
2681 }
2682
2683 // Should the Matcher clone shifts on addressing modes, expecting them
2684 // to be subsumed into complex addressing expressions or compute them
2685 // into registers?
2686 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2687
2688 // Loads and stores with indirect memory input (e.g., volatile loads and
2689 // stores) do not subsume the input into complex addressing expressions. If
2690 // the addressing expression is input to at least one such load or store, do
2691 // not clone the addressing expression. Query needs_acquiring_load and
2692 // needs_releasing_store as a proxy for indirect memory input, as it is not
2693 // possible to directly query for indirect memory input at this stage.
2694 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2695 Node* n = m->fast_out(i);
2696 if (n->is_Load() && needs_acquiring_load(n)) {
2697 return false;
2698 }
2699 if (n->is_Store() && needs_releasing_store(n)) {
2700 return false;
2701 }
2702 }
2703
2704 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2705 return true;
2706 }
2707
2708 Node *off = m->in(AddPNode::Offset);
2709 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2710 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2711 // Are there other uses besides address expressions?
2712 !is_visited(off)) {
2713 address_visited.set(off->_idx); // Flag as address_visited
2714 mstack.push(off->in(2), Visit);
2715 Node *conv = off->in(1);
2716 if (conv->Opcode() == Op_ConvI2L &&
2717 // Are there other uses besides address expressions?
2718 !is_visited(conv)) {
2719 address_visited.set(conv->_idx); // Flag as address_visited
2720 mstack.push(conv->in(1), Pre_Visit);
2721 } else {
2722 mstack.push(conv, Pre_Visit);
2723 }
2724 address_visited.test_set(m->_idx); // Flag as address_visited
2725 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2726 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2727 return true;
2728 } else if (off->Opcode() == Op_ConvI2L &&
2729 // Are there other uses besides address expressions?
2730 !is_visited(off)) {
2731 address_visited.test_set(m->_idx); // Flag as address_visited
2732 address_visited.set(off->_idx); // Flag as address_visited
2733 mstack.push(off->in(1), Pre_Visit);
2734 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2735 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2736 return true;
2737 }
2738 return false;
2739 }
2740
2741 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2742 { \
2743 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2744 guarantee(DISP == 0, "mode not permitted for volatile"); \
2745 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2746 __ INSN(REG, as_Register(BASE)); \
2747 }
2748
2749
2750 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2751 {
2752 Address::extend scale;
2753
2754 // Hooboy, this is fugly. We need a way to communicate to the
2755 // encoder that the index needs to be sign extended, so we have to
2756 // enumerate all the cases.
2757 switch (opcode) {
2758 case INDINDEXSCALEDI2L:
2759 case INDINDEXSCALEDI2LN:
2760 case INDINDEXI2L:
2761 case INDINDEXI2LN:
2762 scale = Address::sxtw(size);
2763 break;
2764 default:
2765 scale = Address::lsl(size);
2766 }
2767
2768 if (index == -1) {
2769 return Address(base, disp);
2770 } else {
2771 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2772 return Address(base, as_Register(index), scale);
2773 }
2774 }
2775
2776
2777 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2778 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2779 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2780 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2781 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2782
2783 // Used for all non-volatile memory accesses. The use of
2784 // $mem->opcode() to discover whether this pattern uses sign-extended
2785 // offsets is something of a kludge.
2786 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2787 Register reg, int opcode,
2788 Register base, int index, int scale, int disp,
2789 int size_in_memory)
2790 {
2791 Address addr = mem2address(opcode, base, index, scale, disp);
2792 if (addr.getMode() == Address::base_plus_offset) {
2793 /* Fix up any out-of-range offsets. */
2794 assert_different_registers(rscratch1, base);
2795 assert_different_registers(rscratch1, reg);
2796 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2797 }
2798 (masm->*insn)(reg, addr);
2799 }
2800
2801 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2802 FloatRegister reg, int opcode,
2803 Register base, int index, int size, int disp,
2804 int size_in_memory)
2805 {
2806 Address::extend scale;
2807
2808 switch (opcode) {
2809 case INDINDEXSCALEDI2L:
2810 case INDINDEXSCALEDI2LN:
2811 scale = Address::sxtw(size);
2812 break;
2813 default:
2814 scale = Address::lsl(size);
2815 }
2816
2817 if (index == -1) {
2818 // Fix up any out-of-range offsets.
2819 assert_different_registers(rscratch1, base);
2820 Address addr = Address(base, disp);
2821 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2822 (masm->*insn)(reg, addr);
2823 } else {
2824 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2825 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2826 }
2827 }
2828
2829 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2830 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2831 int opcode, Register base, int index, int size, int disp)
2832 {
2833 if (index == -1) {
2834 (masm->*insn)(reg, T, Address(base, disp));
2835 } else {
2836 assert(disp == 0, "unsupported address mode");
2837 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2838 }
2839 }
2840
2841 %}
2842
2843
2844
2845 //----------ENCODING BLOCK-----------------------------------------------------
2846 // This block specifies the encoding classes used by the compiler to
2847 // output byte streams. Encoding classes are parameterized macros
2848 // used by Machine Instruction Nodes in order to generate the bit
2849 // encoding of the instruction. Operands specify their base encoding
2850 // interface with the interface keyword. There are currently
2851 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2852 // COND_INTER. REG_INTER causes an operand to generate a function
2853 // which returns its register number when queried. CONST_INTER causes
2854 // an operand to generate a function which returns the value of the
2855 // constant when queried. MEMORY_INTER causes an operand to generate
2856 // four functions which return the Base Register, the Index Register,
2857 // the Scale Value, and the Offset Value of the operand when queried.
2858 // COND_INTER causes an operand to generate six functions which return
2859 // the encoding code (ie - encoding bits for the instruction)
2860 // associated with each basic boolean condition for a conditional
2861 // instruction.
2862 //
2863 // Instructions specify two basic values for encoding. Again, a
2864 // function is available to check if the constant displacement is an
2865 // oop. They use the ins_encode keyword to specify their encoding
2866 // classes (which must be a sequence of enc_class names, and their
2867 // parameters, specified in the encoding block), and they use the
2868 // opcode keyword to specify, in order, their primary, secondary, and
2869 // tertiary opcode. Only the opcode sections which a particular
2870 // instruction needs for encoding need to be specified.
2871 encode %{
2872 // Build emit functions for each basic byte or larger field in the
2873 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2874 // from C++ code in the enc_class source block. Emit functions will
2875 // live in the main source block for now. In future, we can
2876 // generalize this by adding a syntax that specifies the sizes of
2877 // fields in an order, so that the adlc can build the emit functions
2878 // automagically
2879
2880 // catch all for unimplemented encodings
2881 enc_class enc_unimplemented %{
2882 __ unimplemented("C2 catch all");
2883 %}
2884
2885 // BEGIN Non-volatile memory access
2886
2887 // This encoding class is generated automatically from ad_encode.m4.
2888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2889 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2890 Register dst_reg = as_Register($dst$$reg);
2891 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2892 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2893 %}
2894
2895 // This encoding class is generated automatically from ad_encode.m4.
2896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2897 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2898 Register dst_reg = as_Register($dst$$reg);
2899 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2900 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2901 %}
2902
2903 // This encoding class is generated automatically from ad_encode.m4.
2904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2905 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2906 Register dst_reg = as_Register($dst$$reg);
2907 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2908 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2909 %}
2910
2911 // This encoding class is generated automatically from ad_encode.m4.
2912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2913 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2914 Register dst_reg = as_Register($dst$$reg);
2915 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2916 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2917 %}
2918
2919 // This encoding class is generated automatically from ad_encode.m4.
2920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2921 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2922 Register dst_reg = as_Register($dst$$reg);
2923 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2924 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2925 %}
2926
2927 // This encoding class is generated automatically from ad_encode.m4.
2928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2929 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2930 Register dst_reg = as_Register($dst$$reg);
2931 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2932 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2933 %}
2934
2935 // This encoding class is generated automatically from ad_encode.m4.
2936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2937 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2938 Register dst_reg = as_Register($dst$$reg);
2939 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2940 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2941 %}
2942
2943 // This encoding class is generated automatically from ad_encode.m4.
2944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2945 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2946 Register dst_reg = as_Register($dst$$reg);
2947 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2948 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2949 %}
2950
2951 // This encoding class is generated automatically from ad_encode.m4.
2952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2953 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2954 Register dst_reg = as_Register($dst$$reg);
2955 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2956 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2957 %}
2958
2959 // This encoding class is generated automatically from ad_encode.m4.
2960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2961 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2962 Register dst_reg = as_Register($dst$$reg);
2963 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2964 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2965 %}
2966
2967 // This encoding class is generated automatically from ad_encode.m4.
2968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2969 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2970 Register dst_reg = as_Register($dst$$reg);
2971 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2972 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2973 %}
2974
2975 // This encoding class is generated automatically from ad_encode.m4.
2976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2977 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2978 Register dst_reg = as_Register($dst$$reg);
2979 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2980 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2981 %}
2982
2983 // This encoding class is generated automatically from ad_encode.m4.
2984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2985 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2986 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2987 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2988 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2989 %}
2990
2991 // This encoding class is generated automatically from ad_encode.m4.
2992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2993 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2994 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2995 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2996 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2997 %}
2998
2999 // This encoding class is generated automatically from ad_encode.m4.
3000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3001 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3002 Register src_reg = as_Register($src$$reg);
3003 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3004 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3005 %}
3006
3007 // This encoding class is generated automatically from ad_encode.m4.
3008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3009 enc_class aarch64_enc_strb0(memory1 mem) %{
3010 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3011 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3012 %}
3013
3014 // This encoding class is generated automatically from ad_encode.m4.
3015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3016 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3017 Register src_reg = as_Register($src$$reg);
3018 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3019 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3020 %}
3021
3022 // This encoding class is generated automatically from ad_encode.m4.
3023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3024 enc_class aarch64_enc_strh0(memory2 mem) %{
3025 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3026 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3027 %}
3028
3029 // This encoding class is generated automatically from ad_encode.m4.
3030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3031 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3032 Register src_reg = as_Register($src$$reg);
3033 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3034 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3035 %}
3036
3037 // This encoding class is generated automatically from ad_encode.m4.
3038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3039 enc_class aarch64_enc_strw0(memory4 mem) %{
3040 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3041 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3042 %}
3043
3044 // This encoding class is generated automatically from ad_encode.m4.
3045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3046 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3047 Register src_reg = as_Register($src$$reg);
3048 // we sometimes get asked to store the stack pointer into the
3049 // current thread -- we cannot do that directly on AArch64
3050 if (src_reg == r31_sp) {
3051 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3052 __ mov(rscratch2, sp);
3053 src_reg = rscratch2;
3054 }
3055 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3056 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3057 %}
3058
3059 // This encoding class is generated automatically from ad_encode.m4.
3060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3061 enc_class aarch64_enc_str0(memory8 mem) %{
3062 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3063 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3064 %}
3065
3066 // This encoding class is generated automatically from ad_encode.m4.
3067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3068 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3069 FloatRegister src_reg = as_FloatRegister($src$$reg);
3070 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3071 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3072 %}
3073
3074 // This encoding class is generated automatically from ad_encode.m4.
3075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3076 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3077 FloatRegister src_reg = as_FloatRegister($src$$reg);
3078 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3079 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3080 %}
3081
3082 // This encoding class is generated automatically from ad_encode.m4.
3083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3084 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3085 __ membar(Assembler::StoreStore);
3086 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3087 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3088 %}
3089
3090 // END Non-volatile memory access
3091
3092 // Vector loads and stores
3093 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3094 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3095 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3096 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3097 %}
3098
3099 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3100 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3101 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3102 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3103 %}
3104
3105 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3106 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3107 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3108 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3109 %}
3110
3111 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3112 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3113 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3114 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3115 %}
3116
3117 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3118 FloatRegister src_reg = as_FloatRegister($src$$reg);
3119 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3120 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3121 %}
3122
3123 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3124 FloatRegister src_reg = as_FloatRegister($src$$reg);
3125 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3126 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3127 %}
3128
3129 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3130 FloatRegister src_reg = as_FloatRegister($src$$reg);
3131 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3132 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3133 %}
3134
3135 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3136 FloatRegister src_reg = as_FloatRegister($src$$reg);
3137 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3138 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3139 %}
3140
3141 // volatile loads and stores
3142
3143 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3144 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3145 rscratch1, stlrb);
3146 %}
3147
3148 enc_class aarch64_enc_stlrb0(memory mem) %{
3149 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3150 rscratch1, stlrb);
3151 %}
3152
3153 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3154 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, stlrh);
3156 %}
3157
3158 enc_class aarch64_enc_stlrh0(memory mem) %{
3159 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, stlrh);
3161 %}
3162
3163 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3164 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3165 rscratch1, stlrw);
3166 %}
3167
3168 enc_class aarch64_enc_stlrw0(memory mem) %{
3169 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3170 rscratch1, stlrw);
3171 %}
3172
3173 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3174 Register dst_reg = as_Register($dst$$reg);
3175 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, ldarb);
3177 __ sxtbw(dst_reg, dst_reg);
3178 %}
3179
3180 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3181 Register dst_reg = as_Register($dst$$reg);
3182 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3183 rscratch1, ldarb);
3184 __ sxtb(dst_reg, dst_reg);
3185 %}
3186
3187 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3188 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, ldarb);
3190 %}
3191
3192 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3193 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, ldarb);
3195 %}
3196
3197 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3198 Register dst_reg = as_Register($dst$$reg);
3199 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3200 rscratch1, ldarh);
3201 __ sxthw(dst_reg, dst_reg);
3202 %}
3203
3204 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3205 Register dst_reg = as_Register($dst$$reg);
3206 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3207 rscratch1, ldarh);
3208 __ sxth(dst_reg, dst_reg);
3209 %}
3210
3211 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3212 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3213 rscratch1, ldarh);
3214 %}
3215
3216 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3217 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3218 rscratch1, ldarh);
3219 %}
3220
3221 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3222 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3223 rscratch1, ldarw);
3224 %}
3225
3226 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3227 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3228 rscratch1, ldarw);
3229 %}
3230
3231 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3232 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3233 rscratch1, ldar);
3234 %}
3235
3236 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3237 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3238 rscratch1, ldarw);
3239 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3240 %}
3241
3242 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3243 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3244 rscratch1, ldar);
3245 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3246 %}
3247
3248 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3249 Register src_reg = as_Register($src$$reg);
3250 // we sometimes get asked to store the stack pointer into the
3251 // current thread -- we cannot do that directly on AArch64
3252 if (src_reg == r31_sp) {
3253 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3254 __ mov(rscratch2, sp);
3255 src_reg = rscratch2;
3256 }
3257 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3258 rscratch1, stlr);
3259 %}
3260
3261 enc_class aarch64_enc_stlr0(memory mem) %{
3262 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3263 rscratch1, stlr);
3264 %}
3265
3266 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3267 {
3268 FloatRegister src_reg = as_FloatRegister($src$$reg);
3269 __ fmovs(rscratch2, src_reg);
3270 }
3271 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3272 rscratch1, stlrw);
3273 %}
3274
3275 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3276 {
3277 FloatRegister src_reg = as_FloatRegister($src$$reg);
3278 __ fmovd(rscratch2, src_reg);
3279 }
3280 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3281 rscratch1, stlr);
3282 %}
3283
3284 // synchronized read/update encodings
3285
3286 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3287 Register dst_reg = as_Register($dst$$reg);
3288 Register base = as_Register($mem$$base);
3289 int index = $mem$$index;
3290 int scale = $mem$$scale;
3291 int disp = $mem$$disp;
3292 if (index == -1) {
3293 if (disp != 0) {
3294 __ lea(rscratch1, Address(base, disp));
3295 __ ldaxr(dst_reg, rscratch1);
3296 } else {
3297 // TODO
3298 // should we ever get anything other than this case?
3299 __ ldaxr(dst_reg, base);
3300 }
3301 } else {
3302 Register index_reg = as_Register(index);
3303 if (disp == 0) {
3304 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3305 __ ldaxr(dst_reg, rscratch1);
3306 } else {
3307 __ lea(rscratch1, Address(base, disp));
3308 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3309 __ ldaxr(dst_reg, rscratch1);
3310 }
3311 }
3312 %}
3313
3314 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3315 Register src_reg = as_Register($src$$reg);
3316 Register base = as_Register($mem$$base);
3317 int index = $mem$$index;
3318 int scale = $mem$$scale;
3319 int disp = $mem$$disp;
3320 if (index == -1) {
3321 if (disp != 0) {
3322 __ lea(rscratch2, Address(base, disp));
3323 __ stlxr(rscratch1, src_reg, rscratch2);
3324 } else {
3325 // TODO
3326 // should we ever get anything other than this case?
3327 __ stlxr(rscratch1, src_reg, base);
3328 }
3329 } else {
3330 Register index_reg = as_Register(index);
3331 if (disp == 0) {
3332 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3333 __ stlxr(rscratch1, src_reg, rscratch2);
3334 } else {
3335 __ lea(rscratch2, Address(base, disp));
3336 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3337 __ stlxr(rscratch1, src_reg, rscratch2);
3338 }
3339 }
3340 __ cmpw(rscratch1, zr);
3341 %}
3342
3343 // prefetch encodings
3344
3345 enc_class aarch64_enc_prefetchw(memory mem) %{
3346 Register base = as_Register($mem$$base);
3347 int index = $mem$$index;
3348 int scale = $mem$$scale;
3349 int disp = $mem$$disp;
3350 if (index == -1) {
3351 // Fix up any out-of-range offsets.
3352 assert_different_registers(rscratch1, base);
3353 Address addr = Address(base, disp);
3354 addr = __ legitimize_address(addr, 8, rscratch1);
3355 __ prfm(addr, PSTL1KEEP);
3356 } else {
3357 Register index_reg = as_Register(index);
3358 if (disp == 0) {
3359 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3360 } else {
3361 __ lea(rscratch1, Address(base, disp));
3362 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3363 }
3364 }
3365 %}
3366
3367 // mov encodings
3368
3369 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3370 uint32_t con = (uint32_t)$src$$constant;
3371 Register dst_reg = as_Register($dst$$reg);
3372 if (con == 0) {
3373 __ movw(dst_reg, zr);
3374 } else {
3375 __ movw(dst_reg, con);
3376 }
3377 %}
3378
3379 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3380 Register dst_reg = as_Register($dst$$reg);
3381 uint64_t con = (uint64_t)$src$$constant;
3382 if (con == 0) {
3383 __ mov(dst_reg, zr);
3384 } else {
3385 __ mov(dst_reg, con);
3386 }
3387 %}
3388
3389 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3390 Register dst_reg = as_Register($dst$$reg);
3391 address con = (address)$src$$constant;
3392 if (con == nullptr || con == (address)1) {
3393 ShouldNotReachHere();
3394 } else {
3395 relocInfo::relocType rtype = $src->constant_reloc();
3396 if (rtype == relocInfo::oop_type) {
3397 __ movoop(dst_reg, (jobject)con);
3398 } else if (rtype == relocInfo::metadata_type) {
3399 __ mov_metadata(dst_reg, (Metadata*)con);
3400 } else {
3401 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3402 // load fake address constants using a normal move
3403 if (! __ is_valid_AArch64_address(con) ||
3404 con < (address)(uintptr_t)os::vm_page_size() ||
3405 rtype == relocInfo::none) {
3406 __ mov(dst_reg, con);
3407 } else {
3408 // use shorter adrp/add sequence for external_word relocation
3409 uint64_t offset;
3410 __ adrp(dst_reg, Address(con, rtype), offset);
3411 __ add(dst_reg, dst_reg, offset);
3412 }
3413 }
3414 }
3415 %}
3416
3417 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3418 Register dst_reg = as_Register($dst$$reg);
3419 __ mov(dst_reg, zr);
3420 %}
3421
3422 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3423 Register dst_reg = as_Register($dst$$reg);
3424 __ mov(dst_reg, (uint64_t)1);
3425 %}
3426
3427 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3428 Register dst_reg = as_Register($dst$$reg);
3429 address con = (address)$src$$constant;
3430 if (con == nullptr) {
3431 ShouldNotReachHere();
3432 } else {
3433 relocInfo::relocType rtype = $src->constant_reloc();
3434 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3435 __ set_narrow_oop(dst_reg, (jobject)con);
3436 }
3437 %}
3438
3439 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3440 Register dst_reg = as_Register($dst$$reg);
3441 __ mov(dst_reg, zr);
3442 %}
3443
3444 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3445 Register dst_reg = as_Register($dst$$reg);
3446 address con = (address)$src$$constant;
3447 if (con == nullptr) {
3448 ShouldNotReachHere();
3449 } else {
3450 relocInfo::relocType rtype = $src->constant_reloc();
3451 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3452 __ set_narrow_klass(dst_reg, (Klass *)con);
3453 }
3454 %}
3455
3456 // arithmetic encodings
3457
3458 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3459 Register dst_reg = as_Register($dst$$reg);
3460 Register src_reg = as_Register($src1$$reg);
3461 int32_t con = (int32_t)$src2$$constant;
3462 // add has primary == 0, subtract has primary == 1
3463 if ($primary) { con = -con; }
3464 if (con < 0) {
3465 __ subw(dst_reg, src_reg, -con);
3466 } else {
3467 __ addw(dst_reg, src_reg, con);
3468 }
3469 %}
3470
3471 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3472 Register dst_reg = as_Register($dst$$reg);
3473 Register src_reg = as_Register($src1$$reg);
3474 int32_t con = (int32_t)$src2$$constant;
3475 // add has primary == 0, subtract has primary == 1
3476 if ($primary) { con = -con; }
3477 if (con < 0) {
3478 __ sub(dst_reg, src_reg, -con);
3479 } else {
3480 __ add(dst_reg, src_reg, con);
3481 }
3482 %}
3483
3484 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3485 Register dst_reg = as_Register($dst$$reg);
3486 Register src1_reg = as_Register($src1$$reg);
3487 Register src2_reg = as_Register($src2$$reg);
3488 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3489 %}
3490
3491 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3492 Register dst_reg = as_Register($dst$$reg);
3493 Register src1_reg = as_Register($src1$$reg);
3494 Register src2_reg = as_Register($src2$$reg);
3495 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3496 %}
3497
3498 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3499 Register dst_reg = as_Register($dst$$reg);
3500 Register src1_reg = as_Register($src1$$reg);
3501 Register src2_reg = as_Register($src2$$reg);
3502 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3503 %}
3504
3505 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3506 Register dst_reg = as_Register($dst$$reg);
3507 Register src1_reg = as_Register($src1$$reg);
3508 Register src2_reg = as_Register($src2$$reg);
3509 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3510 %}
3511
3512 // compare instruction encodings
3513
3514 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3515 Register reg1 = as_Register($src1$$reg);
3516 Register reg2 = as_Register($src2$$reg);
3517 __ cmpw(reg1, reg2);
3518 %}
3519
3520 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3521 Register reg = as_Register($src1$$reg);
3522 int32_t val = $src2$$constant;
3523 if (val >= 0) {
3524 __ subsw(zr, reg, val);
3525 } else {
3526 __ addsw(zr, reg, -val);
3527 }
3528 %}
3529
3530 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3531 Register reg1 = as_Register($src1$$reg);
3532 uint32_t val = (uint32_t)$src2$$constant;
3533 __ movw(rscratch1, val);
3534 __ cmpw(reg1, rscratch1);
3535 %}
3536
3537 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3538 Register reg1 = as_Register($src1$$reg);
3539 Register reg2 = as_Register($src2$$reg);
3540 __ cmp(reg1, reg2);
3541 %}
3542
3543 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3544 Register reg = as_Register($src1$$reg);
3545 int64_t val = $src2$$constant;
3546 if (val >= 0) {
3547 __ subs(zr, reg, val);
3548 } else if (val != -val) {
3549 __ adds(zr, reg, -val);
3550 } else {
3551 // aargh, Long.MIN_VALUE is a special case
3552 __ orr(rscratch1, zr, (uint64_t)val);
3553 __ subs(zr, reg, rscratch1);
3554 }
3555 %}
3556
3557 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3558 Register reg1 = as_Register($src1$$reg);
3559 uint64_t val = (uint64_t)$src2$$constant;
3560 __ mov(rscratch1, val);
3561 __ cmp(reg1, rscratch1);
3562 %}
3563
3564 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3565 Register reg1 = as_Register($src1$$reg);
3566 Register reg2 = as_Register($src2$$reg);
3567 __ cmp(reg1, reg2);
3568 %}
3569
3570 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3571 Register reg1 = as_Register($src1$$reg);
3572 Register reg2 = as_Register($src2$$reg);
3573 __ cmpw(reg1, reg2);
3574 %}
3575
3576 enc_class aarch64_enc_testp(iRegP src) %{
3577 Register reg = as_Register($src$$reg);
3578 __ cmp(reg, zr);
3579 %}
3580
3581 enc_class aarch64_enc_testn(iRegN src) %{
3582 Register reg = as_Register($src$$reg);
3583 __ cmpw(reg, zr);
3584 %}
3585
3586 enc_class aarch64_enc_b(label lbl) %{
3587 Label *L = $lbl$$label;
3588 __ b(*L);
3589 %}
3590
3591 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3592 Label *L = $lbl$$label;
3593 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3594 %}
3595
3596 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3597 Label *L = $lbl$$label;
3598 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3599 %}
3600
3601 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3602 %{
3603 Register sub_reg = as_Register($sub$$reg);
3604 Register super_reg = as_Register($super$$reg);
3605 Register temp_reg = as_Register($temp$$reg);
3606 Register result_reg = as_Register($result$$reg);
3607
3608 Label miss;
3609 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3610 nullptr, &miss,
3611 /*set_cond_codes:*/ true);
3612 if ($primary) {
3613 __ mov(result_reg, zr);
3614 }
3615 __ bind(miss);
3616 %}
3617
3618 enc_class aarch64_enc_java_static_call(method meth) %{
3619 address addr = (address)$meth$$method;
3620 address call;
3621 if (!_method) {
3622 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3623 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3624 if (call == nullptr) {
3625 ciEnv::current()->record_failure("CodeCache is full");
3626 return;
3627 }
3628 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3629 // The NOP here is purely to ensure that eliding a call to
3630 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3631 __ nop();
3632 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3633 } else {
3634 int method_index = resolved_method_index(masm);
3635 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3636 : static_call_Relocation::spec(method_index);
3637 call = __ trampoline_call(Address(addr, rspec));
3638 if (call == nullptr) {
3639 ciEnv::current()->record_failure("CodeCache is full");
3640 return;
3641 }
3642 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3643 // Calls of the same statically bound method can share
3644 // a stub to the interpreter.
3645 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3646 } else {
3647 // Emit stub for static call
3648 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3649 if (stub == nullptr) {
3650 ciEnv::current()->record_failure("CodeCache is full");
3651 return;
3652 }
3653 }
3654 }
3655
3656 __ post_call_nop();
3657
3658 // Only non uncommon_trap calls need to reinitialize ptrue.
3659 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3660 __ reinitialize_ptrue();
3661 }
3662 %}
3663
3664 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3665 int method_index = resolved_method_index(masm);
3666 address call = __ ic_call((address)$meth$$method, method_index);
3667 if (call == nullptr) {
3668 ciEnv::current()->record_failure("CodeCache is full");
3669 return;
3670 }
3671 __ post_call_nop();
3672 if (Compile::current()->max_vector_size() > 0) {
3673 __ reinitialize_ptrue();
3674 }
3675 %}
3676
3677 enc_class aarch64_enc_call_epilog() %{
3678 if (VerifyStackAtCalls) {
3679 // Check that stack depth is unchanged: find majik cookie on stack
3680 __ call_Unimplemented();
3681 }
3682 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3683 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3684 // Search for the corresponding projection, get the register and emit code that initializes it.
3685 uint con = (tf()->range_cc()->cnt() - 1);
3686 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3687 ProjNode* proj = fast_out(i)->as_Proj();
3688 if (proj->_con == con) {
3689 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3690 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3691 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3692 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3693 __ cmp(r0, zr);
3694 __ cset(toReg, Assembler::NE);
3695 if (reg->is_stack()) {
3696 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3697 __ str(toReg, Address(sp, st_off));
3698 }
3699 break;
3700 }
3701 }
3702 if (return_value_is_used()) {
3703 // An inline type is returned as fields in multiple registers.
3704 // R0 either contains an oop if the inline type is buffered or a pointer
3705 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3706 // if the lowest bit is set to allow C2 to use the oop after null checking.
3707 // r0 &= (r0 & 1) - 1
3708 __ andr(rscratch1, r0, 0x1);
3709 __ sub(rscratch1, rscratch1, 0x1);
3710 __ andr(r0, r0, rscratch1);
3711 }
3712 }
3713 %}
3714
3715 enc_class aarch64_enc_java_to_runtime(method meth) %{
3716 // some calls to generated routines (arraycopy code) are scheduled
3717 // by C2 as runtime calls. if so we can call them using a br (they
3718 // will be in a reachable segment) otherwise we have to use a blr
3719 // which loads the absolute address into a register.
3720 address entry = (address)$meth$$method;
3721 CodeBlob *cb = CodeCache::find_blob(entry);
3722 if (cb) {
3723 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3724 if (call == nullptr) {
3725 ciEnv::current()->record_failure("CodeCache is full");
3726 return;
3727 }
3728 __ post_call_nop();
3729 } else {
3730 Label retaddr;
3731 // Make the anchor frame walkable
3732 __ adr(rscratch2, retaddr);
3733 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3734 __ lea(rscratch1, RuntimeAddress(entry));
3735 __ blr(rscratch1);
3736 __ bind(retaddr);
3737 __ post_call_nop();
3738 }
3739 if (Compile::current()->max_vector_size() > 0) {
3740 __ reinitialize_ptrue();
3741 }
3742 %}
3743
3744 enc_class aarch64_enc_rethrow() %{
3745 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3746 %}
3747
3748 enc_class aarch64_enc_ret() %{
3749 #ifdef ASSERT
3750 if (Compile::current()->max_vector_size() > 0) {
3751 __ verify_ptrue();
3752 }
3753 #endif
3754 __ ret(lr);
3755 %}
3756
3757 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3758 Register target_reg = as_Register($jump_target$$reg);
3759 __ br(target_reg);
3760 %}
3761
3762 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3763 Register target_reg = as_Register($jump_target$$reg);
3764 // exception oop should be in r0
3765 // ret addr has been popped into lr
3766 // callee expects it in r3
3767 __ mov(r3, lr);
3768 __ br(target_reg);
3769 %}
3770
3771 %}
3772
3773 //----------FRAME--------------------------------------------------------------
3774 // Definition of frame structure and management information.
3775 //
3776 // S T A C K L A Y O U T Allocators stack-slot number
3777 // | (to get allocators register number
3778 // G Owned by | | v add OptoReg::stack0())
3779 // r CALLER | |
3780 // o | +--------+ pad to even-align allocators stack-slot
3781 // w V | pad0 | numbers; owned by CALLER
3782 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3783 // h ^ | in | 5
3784 // | | args | 4 Holes in incoming args owned by SELF
3785 // | | | | 3
3786 // | | +--------+
3787 // V | | old out| Empty on Intel, window on Sparc
3788 // | old |preserve| Must be even aligned.
3789 // | SP-+--------+----> Matcher::_old_SP, even aligned
3790 // | | in | 3 area for Intel ret address
3791 // Owned by |preserve| Empty on Sparc.
3792 // SELF +--------+
3793 // | | pad2 | 2 pad to align old SP
3794 // | +--------+ 1
3795 // | | locks | 0
3796 // | +--------+----> OptoReg::stack0(), even aligned
3797 // | | pad1 | 11 pad to align new SP
3798 // | +--------+
3799 // | | | 10
3800 // | | spills | 9 spills
3801 // V | | 8 (pad0 slot for callee)
3802 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3803 // ^ | out | 7
3804 // | | args | 6 Holes in outgoing args owned by CALLEE
3805 // Owned by +--------+
3806 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3807 // | new |preserve| Must be even-aligned.
3808 // | SP-+--------+----> Matcher::_new_SP, even aligned
3809 // | | |
3810 //
3811 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3812 // known from SELF's arguments and the Java calling convention.
3813 // Region 6-7 is determined per call site.
3814 // Note 2: If the calling convention leaves holes in the incoming argument
3815 // area, those holes are owned by SELF. Holes in the outgoing area
3816 // are owned by the CALLEE. Holes should not be necessary in the
3817 // incoming area, as the Java calling convention is completely under
3818 // the control of the AD file. Doubles can be sorted and packed to
3819 // avoid holes. Holes in the outgoing arguments may be necessary for
3820 // varargs C calling conventions.
3821 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3822 // even aligned with pad0 as needed.
3823 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3824 // (the latter is true on Intel but is it false on AArch64?)
3825 // region 6-11 is even aligned; it may be padded out more so that
3826 // the region from SP to FP meets the minimum stack alignment.
3827 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3828 // alignment. Region 11, pad1, may be dynamically extended so that
3829 // SP meets the minimum alignment.
3830
3831 frame %{
3832 // These three registers define part of the calling convention
3833 // between compiled code and the interpreter.
3834
3835 // Inline Cache Register or Method for I2C.
3836 inline_cache_reg(R12);
3837
3838 // Number of stack slots consumed by locking an object
3839 sync_stack_slots(2);
3840
3841 // Compiled code's Frame Pointer
3842 frame_pointer(R31);
3843
3844 // Stack alignment requirement
3845 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3846
3847 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3848 // for calls to C. Supports the var-args backing area for register parms.
3849 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3850
3851 // The after-PROLOG location of the return address. Location of
3852 // return address specifies a type (REG or STACK) and a number
3853 // representing the register number (i.e. - use a register name) or
3854 // stack slot.
3855 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3856 // Otherwise, it is above the locks and verification slot and alignment word
3857 // TODO this may well be correct but need to check why that - 2 is there
3858 // ppc port uses 0 but we definitely need to allow for fixed_slots
3859 // which folds in the space used for monitors
3860 return_addr(STACK - 2 +
3861 align_up((Compile::current()->in_preserve_stack_slots() +
3862 Compile::current()->fixed_slots()),
3863 stack_alignment_in_slots()));
3864
3865 // Location of compiled Java return values. Same as C for now.
3866 return_value
3867 %{
3868 // TODO do we allow ideal_reg == Op_RegN???
3869 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3870 "only return normal values");
3871
3872 static const int lo[Op_RegL + 1] = { // enum name
3873 0, // Op_Node
3874 0, // Op_Set
3875 R0_num, // Op_RegN
3876 R0_num, // Op_RegI
3877 R0_num, // Op_RegP
3878 V0_num, // Op_RegF
3879 V0_num, // Op_RegD
3880 R0_num // Op_RegL
3881 };
3882
3883 static const int hi[Op_RegL + 1] = { // enum name
3884 0, // Op_Node
3885 0, // Op_Set
3886 OptoReg::Bad, // Op_RegN
3887 OptoReg::Bad, // Op_RegI
3888 R0_H_num, // Op_RegP
3889 OptoReg::Bad, // Op_RegF
3890 V0_H_num, // Op_RegD
3891 R0_H_num // Op_RegL
3892 };
3893
3894 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3895 %}
3896 %}
3897
3898 //----------ATTRIBUTES---------------------------------------------------------
3899 //----------Operand Attributes-------------------------------------------------
3900 op_attrib op_cost(1); // Required cost attribute
3901
3902 //----------Instruction Attributes---------------------------------------------
3903 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3904 ins_attrib ins_size(32); // Required size attribute (in bits)
3905 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3906 // a non-matching short branch variant
3907 // of some long branch?
3908 ins_attrib ins_alignment(4); // Required alignment attribute (must
3909 // be a power of 2) specifies the
3910 // alignment that some part of the
3911 // instruction (not necessarily the
3912 // start) requires. If > 1, a
3913 // compute_padding() function must be
3914 // provided for the instruction
3915
3916 // Whether this node is expanded during code emission into a sequence of
3917 // instructions and the first instruction can perform an implicit null check.
3918 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3919
3920 //----------OPERANDS-----------------------------------------------------------
3921 // Operand definitions must precede instruction definitions for correct parsing
3922 // in the ADLC because operands constitute user defined types which are used in
3923 // instruction definitions.
3924
3925 //----------Simple Operands----------------------------------------------------
3926
3927 // Integer operands 32 bit
3928 // 32 bit immediate
3929 operand immI()
3930 %{
3931 match(ConI);
3932
3933 op_cost(0);
3934 format %{ %}
3935 interface(CONST_INTER);
3936 %}
3937
3938 // 32 bit zero
3939 operand immI0()
3940 %{
3941 predicate(n->get_int() == 0);
3942 match(ConI);
3943
3944 op_cost(0);
3945 format %{ %}
3946 interface(CONST_INTER);
3947 %}
3948
3949 // 32 bit unit increment
3950 operand immI_1()
3951 %{
3952 predicate(n->get_int() == 1);
3953 match(ConI);
3954
3955 op_cost(0);
3956 format %{ %}
3957 interface(CONST_INTER);
3958 %}
3959
3960 // 32 bit unit decrement
3961 operand immI_M1()
3962 %{
3963 predicate(n->get_int() == -1);
3964 match(ConI);
3965
3966 op_cost(0);
3967 format %{ %}
3968 interface(CONST_INTER);
3969 %}
3970
3971 // Shift values for add/sub extension shift
3972 operand immIExt()
3973 %{
3974 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3975 match(ConI);
3976
3977 op_cost(0);
3978 format %{ %}
3979 interface(CONST_INTER);
3980 %}
3981
3982 operand immI_gt_1()
3983 %{
3984 predicate(n->get_int() > 1);
3985 match(ConI);
3986
3987 op_cost(0);
3988 format %{ %}
3989 interface(CONST_INTER);
3990 %}
3991
3992 operand immI_le_4()
3993 %{
3994 predicate(n->get_int() <= 4);
3995 match(ConI);
3996
3997 op_cost(0);
3998 format %{ %}
3999 interface(CONST_INTER);
4000 %}
4001
4002 operand immI_4()
4003 %{
4004 predicate(n->get_int() == 4);
4005 match(ConI);
4006
4007 op_cost(0);
4008 format %{ %}
4009 interface(CONST_INTER);
4010 %}
4011
4012 operand immI_16()
4013 %{
4014 predicate(n->get_int() == 16);
4015 match(ConI);
4016
4017 op_cost(0);
4018 format %{ %}
4019 interface(CONST_INTER);
4020 %}
4021
4022 operand immI_24()
4023 %{
4024 predicate(n->get_int() == 24);
4025 match(ConI);
4026
4027 op_cost(0);
4028 format %{ %}
4029 interface(CONST_INTER);
4030 %}
4031
4032 operand immI_32()
4033 %{
4034 predicate(n->get_int() == 32);
4035 match(ConI);
4036
4037 op_cost(0);
4038 format %{ %}
4039 interface(CONST_INTER);
4040 %}
4041
4042 operand immI_48()
4043 %{
4044 predicate(n->get_int() == 48);
4045 match(ConI);
4046
4047 op_cost(0);
4048 format %{ %}
4049 interface(CONST_INTER);
4050 %}
4051
4052 operand immI_56()
4053 %{
4054 predicate(n->get_int() == 56);
4055 match(ConI);
4056
4057 op_cost(0);
4058 format %{ %}
4059 interface(CONST_INTER);
4060 %}
4061
4062 operand immI_255()
4063 %{
4064 predicate(n->get_int() == 255);
4065 match(ConI);
4066
4067 op_cost(0);
4068 format %{ %}
4069 interface(CONST_INTER);
4070 %}
4071
4072 operand immI_65535()
4073 %{
4074 predicate(n->get_int() == 65535);
4075 match(ConI);
4076
4077 op_cost(0);
4078 format %{ %}
4079 interface(CONST_INTER);
4080 %}
4081
4082 operand immI_positive()
4083 %{
4084 predicate(n->get_int() > 0);
4085 match(ConI);
4086
4087 op_cost(0);
4088 format %{ %}
4089 interface(CONST_INTER);
4090 %}
4091
4092 // BoolTest condition for signed compare
4093 operand immI_cmp_cond()
4094 %{
4095 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4096 match(ConI);
4097
4098 op_cost(0);
4099 format %{ %}
4100 interface(CONST_INTER);
4101 %}
4102
4103 // BoolTest condition for unsigned compare
4104 operand immI_cmpU_cond()
4105 %{
4106 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4107 match(ConI);
4108
4109 op_cost(0);
4110 format %{ %}
4111 interface(CONST_INTER);
4112 %}
4113
4114 operand immL_255()
4115 %{
4116 predicate(n->get_long() == 255L);
4117 match(ConL);
4118
4119 op_cost(0);
4120 format %{ %}
4121 interface(CONST_INTER);
4122 %}
4123
4124 operand immL_65535()
4125 %{
4126 predicate(n->get_long() == 65535L);
4127 match(ConL);
4128
4129 op_cost(0);
4130 format %{ %}
4131 interface(CONST_INTER);
4132 %}
4133
4134 operand immL_4294967295()
4135 %{
4136 predicate(n->get_long() == 4294967295L);
4137 match(ConL);
4138
4139 op_cost(0);
4140 format %{ %}
4141 interface(CONST_INTER);
4142 %}
4143
4144 operand immL_bitmask()
4145 %{
4146 predicate((n->get_long() != 0)
4147 && ((n->get_long() & 0xc000000000000000l) == 0)
4148 && is_power_of_2(n->get_long() + 1));
4149 match(ConL);
4150
4151 op_cost(0);
4152 format %{ %}
4153 interface(CONST_INTER);
4154 %}
4155
4156 operand immI_bitmask()
4157 %{
4158 predicate((n->get_int() != 0)
4159 && ((n->get_int() & 0xc0000000) == 0)
4160 && is_power_of_2(n->get_int() + 1));
4161 match(ConI);
4162
4163 op_cost(0);
4164 format %{ %}
4165 interface(CONST_INTER);
4166 %}
4167
4168 operand immL_positive_bitmaskI()
4169 %{
4170 predicate((n->get_long() != 0)
4171 && ((julong)n->get_long() < 0x80000000ULL)
4172 && is_power_of_2(n->get_long() + 1));
4173 match(ConL);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 // Scale values for scaled offset addressing modes (up to long but not quad)
4181 operand immIScale()
4182 %{
4183 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4184 match(ConI);
4185
4186 op_cost(0);
4187 format %{ %}
4188 interface(CONST_INTER);
4189 %}
4190
4191 // 5 bit signed integer
4192 operand immI5()
4193 %{
4194 predicate(Assembler::is_simm(n->get_int(), 5));
4195 match(ConI);
4196
4197 op_cost(0);
4198 format %{ %}
4199 interface(CONST_INTER);
4200 %}
4201
4202 // 7 bit unsigned integer
4203 operand immIU7()
4204 %{
4205 predicate(Assembler::is_uimm(n->get_int(), 7));
4206 match(ConI);
4207
4208 op_cost(0);
4209 format %{ %}
4210 interface(CONST_INTER);
4211 %}
4212
4213 // Offset for scaled or unscaled immediate loads and stores
4214 operand immIOffset()
4215 %{
4216 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4217 match(ConI);
4218
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 operand immIOffset1()
4225 %{
4226 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4227 match(ConI);
4228
4229 op_cost(0);
4230 format %{ %}
4231 interface(CONST_INTER);
4232 %}
4233
4234 operand immIOffset2()
4235 %{
4236 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4237 match(ConI);
4238
4239 op_cost(0);
4240 format %{ %}
4241 interface(CONST_INTER);
4242 %}
4243
4244 operand immIOffset4()
4245 %{
4246 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4247 match(ConI);
4248
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 operand immIOffset8()
4255 %{
4256 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4257 match(ConI);
4258
4259 op_cost(0);
4260 format %{ %}
4261 interface(CONST_INTER);
4262 %}
4263
4264 operand immIOffset16()
4265 %{
4266 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4267 match(ConI);
4268
4269 op_cost(0);
4270 format %{ %}
4271 interface(CONST_INTER);
4272 %}
4273
4274 operand immLOffset()
4275 %{
4276 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4277 match(ConL);
4278
4279 op_cost(0);
4280 format %{ %}
4281 interface(CONST_INTER);
4282 %}
4283
4284 operand immLoffset1()
4285 %{
4286 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4287 match(ConL);
4288
4289 op_cost(0);
4290 format %{ %}
4291 interface(CONST_INTER);
4292 %}
4293
4294 operand immLoffset2()
4295 %{
4296 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4297 match(ConL);
4298
4299 op_cost(0);
4300 format %{ %}
4301 interface(CONST_INTER);
4302 %}
4303
4304 operand immLoffset4()
4305 %{
4306 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4307 match(ConL);
4308
4309 op_cost(0);
4310 format %{ %}
4311 interface(CONST_INTER);
4312 %}
4313
4314 operand immLoffset8()
4315 %{
4316 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4317 match(ConL);
4318
4319 op_cost(0);
4320 format %{ %}
4321 interface(CONST_INTER);
4322 %}
4323
4324 operand immLoffset16()
4325 %{
4326 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4327 match(ConL);
4328
4329 op_cost(0);
4330 format %{ %}
4331 interface(CONST_INTER);
4332 %}
4333
4334 // 5 bit signed long integer
4335 operand immL5()
4336 %{
4337 predicate(Assembler::is_simm(n->get_long(), 5));
4338 match(ConL);
4339
4340 op_cost(0);
4341 format %{ %}
4342 interface(CONST_INTER);
4343 %}
4344
4345 // 7 bit unsigned long integer
4346 operand immLU7()
4347 %{
4348 predicate(Assembler::is_uimm(n->get_long(), 7));
4349 match(ConL);
4350
4351 op_cost(0);
4352 format %{ %}
4353 interface(CONST_INTER);
4354 %}
4355
4356 // 8 bit signed value.
4357 operand immI8()
4358 %{
4359 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4360 match(ConI);
4361
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 // 8 bit signed value (simm8), or #simm8 LSL 8.
4368 operand immIDupV()
4369 %{
4370 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4371 match(ConI);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 // 8 bit signed value (simm8), or #simm8 LSL 8.
4379 operand immLDupV()
4380 %{
4381 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4382 match(ConL);
4383
4384 op_cost(0);
4385 format %{ %}
4386 interface(CONST_INTER);
4387 %}
4388
4389 // 8 bit signed value (simm8), or #simm8 LSL 8.
4390 operand immHDupV()
4391 %{
4392 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4393 match(ConH);
4394
4395 op_cost(0);
4396 format %{ %}
4397 interface(CONST_INTER);
4398 %}
4399
4400 // 8 bit integer valid for vector add sub immediate
4401 operand immBAddSubV()
4402 %{
4403 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4404 match(ConI);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 // 32 bit integer valid for add sub immediate
4412 operand immIAddSub()
4413 %{
4414 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4415 match(ConI);
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 // 32 bit integer valid for vector add sub immediate
4422 operand immIAddSubV()
4423 %{
4424 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4425 match(ConI);
4426
4427 op_cost(0);
4428 format %{ %}
4429 interface(CONST_INTER);
4430 %}
4431
4432 // 32 bit unsigned integer valid for logical immediate
4433
4434 operand immBLog()
4435 %{
4436 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4437 match(ConI);
4438
4439 op_cost(0);
4440 format %{ %}
4441 interface(CONST_INTER);
4442 %}
4443
4444 operand immSLog()
4445 %{
4446 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4447 match(ConI);
4448
4449 op_cost(0);
4450 format %{ %}
4451 interface(CONST_INTER);
4452 %}
4453
4454 operand immILog()
4455 %{
4456 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4457 match(ConI);
4458
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 // Integer operands 64 bit
4465 // 64 bit immediate
4466 operand immL()
4467 %{
4468 match(ConL);
4469
4470 op_cost(0);
4471 format %{ %}
4472 interface(CONST_INTER);
4473 %}
4474
4475 // 64 bit zero
4476 operand immL0()
4477 %{
4478 predicate(n->get_long() == 0);
4479 match(ConL);
4480
4481 op_cost(0);
4482 format %{ %}
4483 interface(CONST_INTER);
4484 %}
4485
4486 // 64 bit unit decrement
4487 operand immL_M1()
4488 %{
4489 predicate(n->get_long() == -1);
4490 match(ConL);
4491
4492 op_cost(0);
4493 format %{ %}
4494 interface(CONST_INTER);
4495 %}
4496
4497 // 64 bit integer valid for add sub immediate
4498 operand immLAddSub()
4499 %{
4500 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4501 match(ConL);
4502 op_cost(0);
4503 format %{ %}
4504 interface(CONST_INTER);
4505 %}
4506
4507 // 64 bit integer valid for addv subv immediate
4508 operand immLAddSubV()
4509 %{
4510 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4511 match(ConL);
4512
4513 op_cost(0);
4514 format %{ %}
4515 interface(CONST_INTER);
4516 %}
4517
4518 // 64 bit integer valid for logical immediate
4519 operand immLLog()
4520 %{
4521 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4522 match(ConL);
4523 op_cost(0);
4524 format %{ %}
4525 interface(CONST_INTER);
4526 %}
4527
4528 // Long Immediate: low 32-bit mask
4529 operand immL_32bits()
4530 %{
4531 predicate(n->get_long() == 0xFFFFFFFFL);
4532 match(ConL);
4533 op_cost(0);
4534 format %{ %}
4535 interface(CONST_INTER);
4536 %}
4537
4538 // Pointer operands
4539 // Pointer Immediate
4540 operand immP()
4541 %{
4542 match(ConP);
4543
4544 op_cost(0);
4545 format %{ %}
4546 interface(CONST_INTER);
4547 %}
4548
4549 // nullptr Pointer Immediate
4550 operand immP0()
4551 %{
4552 predicate(n->get_ptr() == 0);
4553 match(ConP);
4554
4555 op_cost(0);
4556 format %{ %}
4557 interface(CONST_INTER);
4558 %}
4559
4560 // Pointer Immediate One
4561 // this is used in object initialization (initial object header)
4562 operand immP_1()
4563 %{
4564 predicate(n->get_ptr() == 1);
4565 match(ConP);
4566
4567 op_cost(0);
4568 format %{ %}
4569 interface(CONST_INTER);
4570 %}
4571
4572 // AOT Runtime Constants Address
4573 operand immAOTRuntimeConstantsAddress()
4574 %{
4575 // Check if the address is in the range of AOT Runtime Constants
4576 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4577 match(ConP);
4578
4579 op_cost(0);
4580 format %{ %}
4581 interface(CONST_INTER);
4582 %}
4583
4584 // Float and Double operands
4585 // Double Immediate
4586 operand immD()
4587 %{
4588 match(ConD);
4589 op_cost(0);
4590 format %{ %}
4591 interface(CONST_INTER);
4592 %}
4593
4594 // Double Immediate: +0.0d
4595 operand immD0()
4596 %{
4597 predicate(jlong_cast(n->getd()) == 0);
4598 match(ConD);
4599
4600 op_cost(0);
4601 format %{ %}
4602 interface(CONST_INTER);
4603 %}
4604
4605 // constant 'double +0.0'.
4606 operand immDPacked()
4607 %{
4608 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4609 match(ConD);
4610 op_cost(0);
4611 format %{ %}
4612 interface(CONST_INTER);
4613 %}
4614
4615 // Float Immediate
4616 operand immF()
4617 %{
4618 match(ConF);
4619 op_cost(0);
4620 format %{ %}
4621 interface(CONST_INTER);
4622 %}
4623
4624 // Float Immediate: +0.0f.
4625 operand immF0()
4626 %{
4627 predicate(jint_cast(n->getf()) == 0);
4628 match(ConF);
4629
4630 op_cost(0);
4631 format %{ %}
4632 interface(CONST_INTER);
4633 %}
4634
4635 // Half Float (FP16) Immediate
4636 operand immH()
4637 %{
4638 match(ConH);
4639 op_cost(0);
4640 format %{ %}
4641 interface(CONST_INTER);
4642 %}
4643
4644 //
4645 operand immFPacked()
4646 %{
4647 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4648 match(ConF);
4649 op_cost(0);
4650 format %{ %}
4651 interface(CONST_INTER);
4652 %}
4653
4654 // Narrow pointer operands
4655 // Narrow Pointer Immediate
4656 operand immN()
4657 %{
4658 match(ConN);
4659
4660 op_cost(0);
4661 format %{ %}
4662 interface(CONST_INTER);
4663 %}
4664
4665 // Narrow nullptr Pointer Immediate
4666 operand immN0()
4667 %{
4668 predicate(n->get_narrowcon() == 0);
4669 match(ConN);
4670
4671 op_cost(0);
4672 format %{ %}
4673 interface(CONST_INTER);
4674 %}
4675
4676 operand immNKlass()
4677 %{
4678 match(ConNKlass);
4679
4680 op_cost(0);
4681 format %{ %}
4682 interface(CONST_INTER);
4683 %}
4684
4685 // Integer 32 bit Register Operands
4686 // Integer 32 bitRegister (excludes SP)
4687 operand iRegI()
4688 %{
4689 constraint(ALLOC_IN_RC(any_reg32));
4690 match(RegI);
4691 match(iRegINoSp);
4692 op_cost(0);
4693 format %{ %}
4694 interface(REG_INTER);
4695 %}
4696
4697 // Integer 32 bit Register not Special
4698 operand iRegINoSp()
4699 %{
4700 constraint(ALLOC_IN_RC(no_special_reg32));
4701 match(RegI);
4702 op_cost(0);
4703 format %{ %}
4704 interface(REG_INTER);
4705 %}
4706
4707 // Integer 64 bit Register Operands
4708 // Integer 64 bit Register (includes SP)
4709 operand iRegL()
4710 %{
4711 constraint(ALLOC_IN_RC(any_reg));
4712 match(RegL);
4713 match(iRegLNoSp);
4714 op_cost(0);
4715 format %{ %}
4716 interface(REG_INTER);
4717 %}
4718
4719 // Integer 64 bit Register not Special
4720 operand iRegLNoSp()
4721 %{
4722 constraint(ALLOC_IN_RC(no_special_reg));
4723 match(RegL);
4724 match(iRegL_R0);
4725 format %{ %}
4726 interface(REG_INTER);
4727 %}
4728
4729 // Pointer Register Operands
4730 // Pointer Register
4731 operand iRegP()
4732 %{
4733 constraint(ALLOC_IN_RC(ptr_reg));
4734 match(RegP);
4735 match(iRegPNoSp);
4736 match(iRegP_R0);
4737 //match(iRegP_R2);
4738 //match(iRegP_R4);
4739 match(iRegP_R5);
4740 match(thread_RegP);
4741 op_cost(0);
4742 format %{ %}
4743 interface(REG_INTER);
4744 %}
4745
4746 // Pointer 64 bit Register not Special
4747 operand iRegPNoSp()
4748 %{
4749 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4750 match(RegP);
4751 // match(iRegP);
4752 // match(iRegP_R0);
4753 // match(iRegP_R2);
4754 // match(iRegP_R4);
4755 // match(iRegP_R5);
4756 // match(thread_RegP);
4757 op_cost(0);
4758 format %{ %}
4759 interface(REG_INTER);
4760 %}
4761
4762 // This operand is not allowed to use rfp even if
4763 // rfp is not used to hold the frame pointer.
4764 operand iRegPNoSpNoRfp()
4765 %{
4766 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4767 match(RegP);
4768 match(iRegPNoSp);
4769 op_cost(0);
4770 format %{ %}
4771 interface(REG_INTER);
4772 %}
4773
4774 // Pointer 64 bit Register R0 only
4775 operand iRegP_R0()
4776 %{
4777 constraint(ALLOC_IN_RC(r0_reg));
4778 match(RegP);
4779 // match(iRegP);
4780 match(iRegPNoSp);
4781 op_cost(0);
4782 format %{ %}
4783 interface(REG_INTER);
4784 %}
4785
4786 // Pointer 64 bit Register R1 only
4787 operand iRegP_R1()
4788 %{
4789 constraint(ALLOC_IN_RC(r1_reg));
4790 match(RegP);
4791 // match(iRegP);
4792 match(iRegPNoSp);
4793 op_cost(0);
4794 format %{ %}
4795 interface(REG_INTER);
4796 %}
4797
4798 // Pointer 64 bit Register R2 only
4799 operand iRegP_R2()
4800 %{
4801 constraint(ALLOC_IN_RC(r2_reg));
4802 match(RegP);
4803 // match(iRegP);
4804 match(iRegPNoSp);
4805 op_cost(0);
4806 format %{ %}
4807 interface(REG_INTER);
4808 %}
4809
4810 // Pointer 64 bit Register R3 only
4811 operand iRegP_R3()
4812 %{
4813 constraint(ALLOC_IN_RC(r3_reg));
4814 match(RegP);
4815 // match(iRegP);
4816 match(iRegPNoSp);
4817 op_cost(0);
4818 format %{ %}
4819 interface(REG_INTER);
4820 %}
4821
4822 // Pointer 64 bit Register R4 only
4823 operand iRegP_R4()
4824 %{
4825 constraint(ALLOC_IN_RC(r4_reg));
4826 match(RegP);
4827 // match(iRegP);
4828 match(iRegPNoSp);
4829 op_cost(0);
4830 format %{ %}
4831 interface(REG_INTER);
4832 %}
4833
4834 // Pointer 64 bit Register R5 only
4835 operand iRegP_R5()
4836 %{
4837 constraint(ALLOC_IN_RC(r5_reg));
4838 match(RegP);
4839 // match(iRegP);
4840 match(iRegPNoSp);
4841 op_cost(0);
4842 format %{ %}
4843 interface(REG_INTER);
4844 %}
4845
4846 // Pointer 64 bit Register R10 only
4847 operand iRegP_R10()
4848 %{
4849 constraint(ALLOC_IN_RC(r10_reg));
4850 match(RegP);
4851 // match(iRegP);
4852 match(iRegPNoSp);
4853 op_cost(0);
4854 format %{ %}
4855 interface(REG_INTER);
4856 %}
4857
4858 // Long 64 bit Register R0 only
4859 operand iRegL_R0()
4860 %{
4861 constraint(ALLOC_IN_RC(r0_reg));
4862 match(RegL);
4863 match(iRegLNoSp);
4864 op_cost(0);
4865 format %{ %}
4866 interface(REG_INTER);
4867 %}
4868
4869 // Long 64 bit Register R11 only
4870 operand iRegL_R11()
4871 %{
4872 constraint(ALLOC_IN_RC(r11_reg));
4873 match(RegL);
4874 match(iRegLNoSp);
4875 op_cost(0);
4876 format %{ %}
4877 interface(REG_INTER);
4878 %}
4879
4880 // Register R0 only
4881 operand iRegI_R0()
4882 %{
4883 constraint(ALLOC_IN_RC(int_r0_reg));
4884 match(RegI);
4885 match(iRegINoSp);
4886 op_cost(0);
4887 format %{ %}
4888 interface(REG_INTER);
4889 %}
4890
4891 // Register R2 only
4892 operand iRegI_R2()
4893 %{
4894 constraint(ALLOC_IN_RC(int_r2_reg));
4895 match(RegI);
4896 match(iRegINoSp);
4897 op_cost(0);
4898 format %{ %}
4899 interface(REG_INTER);
4900 %}
4901
4902 // Register R3 only
4903 operand iRegI_R3()
4904 %{
4905 constraint(ALLOC_IN_RC(int_r3_reg));
4906 match(RegI);
4907 match(iRegINoSp);
4908 op_cost(0);
4909 format %{ %}
4910 interface(REG_INTER);
4911 %}
4912
4913
4914 // Register R4 only
4915 operand iRegI_R4()
4916 %{
4917 constraint(ALLOC_IN_RC(int_r4_reg));
4918 match(RegI);
4919 match(iRegINoSp);
4920 op_cost(0);
4921 format %{ %}
4922 interface(REG_INTER);
4923 %}
4924
4925
4926 // Pointer Register Operands
4927 // Narrow Pointer Register
4928 operand iRegN()
4929 %{
4930 constraint(ALLOC_IN_RC(any_reg32));
4931 match(RegN);
4932 match(iRegNNoSp);
4933 op_cost(0);
4934 format %{ %}
4935 interface(REG_INTER);
4936 %}
4937
4938 // Integer 64 bit Register not Special
4939 operand iRegNNoSp()
4940 %{
4941 constraint(ALLOC_IN_RC(no_special_reg32));
4942 match(RegN);
4943 op_cost(0);
4944 format %{ %}
4945 interface(REG_INTER);
4946 %}
4947
4948 // Float Register
4949 // Float register operands
4950 operand vRegF()
4951 %{
4952 constraint(ALLOC_IN_RC(float_reg));
4953 match(RegF);
4954
4955 op_cost(0);
4956 format %{ %}
4957 interface(REG_INTER);
4958 %}
4959
4960 // Double Register
4961 // Double register operands
4962 operand vRegD()
4963 %{
4964 constraint(ALLOC_IN_RC(double_reg));
4965 match(RegD);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 // Generic vector class. This will be used for
4973 // all vector operands, including NEON and SVE.
4974 operand vReg()
4975 %{
4976 constraint(ALLOC_IN_RC(dynamic));
4977 match(VecA);
4978 match(VecD);
4979 match(VecX);
4980
4981 op_cost(0);
4982 format %{ %}
4983 interface(REG_INTER);
4984 %}
4985
4986 operand vReg_V10()
4987 %{
4988 constraint(ALLOC_IN_RC(v10_veca_reg));
4989 match(vReg);
4990
4991 op_cost(0);
4992 format %{ %}
4993 interface(REG_INTER);
4994 %}
4995
4996 operand vReg_V11()
4997 %{
4998 constraint(ALLOC_IN_RC(v11_veca_reg));
4999 match(vReg);
5000
5001 op_cost(0);
5002 format %{ %}
5003 interface(REG_INTER);
5004 %}
5005
5006 operand vReg_V12()
5007 %{
5008 constraint(ALLOC_IN_RC(v12_veca_reg));
5009 match(vReg);
5010
5011 op_cost(0);
5012 format %{ %}
5013 interface(REG_INTER);
5014 %}
5015
5016 operand vReg_V13()
5017 %{
5018 constraint(ALLOC_IN_RC(v13_veca_reg));
5019 match(vReg);
5020
5021 op_cost(0);
5022 format %{ %}
5023 interface(REG_INTER);
5024 %}
5025
5026 operand vReg_V17()
5027 %{
5028 constraint(ALLOC_IN_RC(v17_veca_reg));
5029 match(vReg);
5030
5031 op_cost(0);
5032 format %{ %}
5033 interface(REG_INTER);
5034 %}
5035
5036 operand vReg_V18()
5037 %{
5038 constraint(ALLOC_IN_RC(v18_veca_reg));
5039 match(vReg);
5040
5041 op_cost(0);
5042 format %{ %}
5043 interface(REG_INTER);
5044 %}
5045
5046 operand vReg_V23()
5047 %{
5048 constraint(ALLOC_IN_RC(v23_veca_reg));
5049 match(vReg);
5050
5051 op_cost(0);
5052 format %{ %}
5053 interface(REG_INTER);
5054 %}
5055
5056 operand vReg_V24()
5057 %{
5058 constraint(ALLOC_IN_RC(v24_veca_reg));
5059 match(vReg);
5060
5061 op_cost(0);
5062 format %{ %}
5063 interface(REG_INTER);
5064 %}
5065
5066 operand vecA()
5067 %{
5068 constraint(ALLOC_IN_RC(vectora_reg));
5069 match(VecA);
5070
5071 op_cost(0);
5072 format %{ %}
5073 interface(REG_INTER);
5074 %}
5075
5076 operand vecD()
5077 %{
5078 constraint(ALLOC_IN_RC(vectord_reg));
5079 match(VecD);
5080
5081 op_cost(0);
5082 format %{ %}
5083 interface(REG_INTER);
5084 %}
5085
5086 operand vecX()
5087 %{
5088 constraint(ALLOC_IN_RC(vectorx_reg));
5089 match(VecX);
5090
5091 op_cost(0);
5092 format %{ %}
5093 interface(REG_INTER);
5094 %}
5095
5096 operand vRegD_V0()
5097 %{
5098 constraint(ALLOC_IN_RC(v0_reg));
5099 match(RegD);
5100 op_cost(0);
5101 format %{ %}
5102 interface(REG_INTER);
5103 %}
5104
5105 operand vRegD_V1()
5106 %{
5107 constraint(ALLOC_IN_RC(v1_reg));
5108 match(RegD);
5109 op_cost(0);
5110 format %{ %}
5111 interface(REG_INTER);
5112 %}
5113
5114 operand vRegD_V2()
5115 %{
5116 constraint(ALLOC_IN_RC(v2_reg));
5117 match(RegD);
5118 op_cost(0);
5119 format %{ %}
5120 interface(REG_INTER);
5121 %}
5122
5123 operand vRegD_V3()
5124 %{
5125 constraint(ALLOC_IN_RC(v3_reg));
5126 match(RegD);
5127 op_cost(0);
5128 format %{ %}
5129 interface(REG_INTER);
5130 %}
5131
5132 operand vRegD_V4()
5133 %{
5134 constraint(ALLOC_IN_RC(v4_reg));
5135 match(RegD);
5136 op_cost(0);
5137 format %{ %}
5138 interface(REG_INTER);
5139 %}
5140
5141 operand vRegD_V5()
5142 %{
5143 constraint(ALLOC_IN_RC(v5_reg));
5144 match(RegD);
5145 op_cost(0);
5146 format %{ %}
5147 interface(REG_INTER);
5148 %}
5149
5150 operand vRegD_V6()
5151 %{
5152 constraint(ALLOC_IN_RC(v6_reg));
5153 match(RegD);
5154 op_cost(0);
5155 format %{ %}
5156 interface(REG_INTER);
5157 %}
5158
5159 operand vRegD_V7()
5160 %{
5161 constraint(ALLOC_IN_RC(v7_reg));
5162 match(RegD);
5163 op_cost(0);
5164 format %{ %}
5165 interface(REG_INTER);
5166 %}
5167
5168 operand vRegD_V12()
5169 %{
5170 constraint(ALLOC_IN_RC(v12_reg));
5171 match(RegD);
5172 op_cost(0);
5173 format %{ %}
5174 interface(REG_INTER);
5175 %}
5176
5177 operand vRegD_V13()
5178 %{
5179 constraint(ALLOC_IN_RC(v13_reg));
5180 match(RegD);
5181 op_cost(0);
5182 format %{ %}
5183 interface(REG_INTER);
5184 %}
5185
5186 operand pReg()
5187 %{
5188 constraint(ALLOC_IN_RC(pr_reg));
5189 match(RegVectMask);
5190 match(pRegGov);
5191 op_cost(0);
5192 format %{ %}
5193 interface(REG_INTER);
5194 %}
5195
5196 operand pRegGov()
5197 %{
5198 constraint(ALLOC_IN_RC(gov_pr));
5199 match(RegVectMask);
5200 match(pReg);
5201 op_cost(0);
5202 format %{ %}
5203 interface(REG_INTER);
5204 %}
5205
5206 operand pRegGov_P0()
5207 %{
5208 constraint(ALLOC_IN_RC(p0_reg));
5209 match(RegVectMask);
5210 op_cost(0);
5211 format %{ %}
5212 interface(REG_INTER);
5213 %}
5214
5215 operand pRegGov_P1()
5216 %{
5217 constraint(ALLOC_IN_RC(p1_reg));
5218 match(RegVectMask);
5219 op_cost(0);
5220 format %{ %}
5221 interface(REG_INTER);
5222 %}
5223
5224 // Flags register, used as output of signed compare instructions
5225
5226 // note that on AArch64 we also use this register as the output for
5227 // for floating point compare instructions (CmpF CmpD). this ensures
5228 // that ordered inequality tests use GT, GE, LT or LE none of which
5229 // pass through cases where the result is unordered i.e. one or both
5230 // inputs to the compare is a NaN. this means that the ideal code can
5231 // replace e.g. a GT with an LE and not end up capturing the NaN case
5232 // (where the comparison should always fail). EQ and NE tests are
5233 // always generated in ideal code so that unordered folds into the NE
5234 // case, matching the behaviour of AArch64 NE.
5235 //
5236 // This differs from x86 where the outputs of FP compares use a
5237 // special FP flags registers and where compares based on this
5238 // register are distinguished into ordered inequalities (cmpOpUCF) and
5239 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5240 // to explicitly handle the unordered case in branches. x86 also has
5241 // to include extra CMoveX rules to accept a cmpOpUCF input.
5242
5243 operand rFlagsReg()
5244 %{
5245 constraint(ALLOC_IN_RC(int_flags));
5246 match(RegFlags);
5247
5248 op_cost(0);
5249 format %{ "RFLAGS" %}
5250 interface(REG_INTER);
5251 %}
5252
5253 // Flags register, used as output of unsigned compare instructions
5254 operand rFlagsRegU()
5255 %{
5256 constraint(ALLOC_IN_RC(int_flags));
5257 match(RegFlags);
5258
5259 op_cost(0);
5260 format %{ "RFLAGSU" %}
5261 interface(REG_INTER);
5262 %}
5263
5264 // Special Registers
5265
5266 // Method Register
5267 operand inline_cache_RegP(iRegP reg)
5268 %{
5269 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5270 match(reg);
5271 match(iRegPNoSp);
5272 op_cost(0);
5273 format %{ %}
5274 interface(REG_INTER);
5275 %}
5276
5277 // Thread Register
5278 operand thread_RegP(iRegP reg)
5279 %{
5280 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5281 match(reg);
5282 op_cost(0);
5283 format %{ %}
5284 interface(REG_INTER);
5285 %}
5286
5287 //----------Memory Operands----------------------------------------------------
5288
5289 operand indirect(iRegP reg)
5290 %{
5291 constraint(ALLOC_IN_RC(ptr_reg));
5292 match(reg);
5293 op_cost(0);
5294 format %{ "[$reg]" %}
5295 interface(MEMORY_INTER) %{
5296 base($reg);
5297 index(0xffffffff);
5298 scale(0x0);
5299 disp(0x0);
5300 %}
5301 %}
5302
5303 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5304 %{
5305 constraint(ALLOC_IN_RC(ptr_reg));
5306 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5307 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5308 op_cost(0);
5309 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5310 interface(MEMORY_INTER) %{
5311 base($reg);
5312 index($ireg);
5313 scale($scale);
5314 disp(0x0);
5315 %}
5316 %}
5317
5318 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5319 %{
5320 constraint(ALLOC_IN_RC(ptr_reg));
5321 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5322 match(AddP reg (LShiftL lreg scale));
5323 op_cost(0);
5324 format %{ "$reg, $lreg lsl($scale)" %}
5325 interface(MEMORY_INTER) %{
5326 base($reg);
5327 index($lreg);
5328 scale($scale);
5329 disp(0x0);
5330 %}
5331 %}
5332
5333 operand indIndexI2L(iRegP reg, iRegI ireg)
5334 %{
5335 constraint(ALLOC_IN_RC(ptr_reg));
5336 match(AddP reg (ConvI2L ireg));
5337 op_cost(0);
5338 format %{ "$reg, $ireg, 0, I2L" %}
5339 interface(MEMORY_INTER) %{
5340 base($reg);
5341 index($ireg);
5342 scale(0x0);
5343 disp(0x0);
5344 %}
5345 %}
5346
5347 operand indIndex(iRegP reg, iRegL lreg)
5348 %{
5349 constraint(ALLOC_IN_RC(ptr_reg));
5350 match(AddP reg lreg);
5351 op_cost(0);
5352 format %{ "$reg, $lreg" %}
5353 interface(MEMORY_INTER) %{
5354 base($reg);
5355 index($lreg);
5356 scale(0x0);
5357 disp(0x0);
5358 %}
5359 %}
5360
5361 operand indOffI1(iRegP reg, immIOffset1 off)
5362 %{
5363 constraint(ALLOC_IN_RC(ptr_reg));
5364 match(AddP reg off);
5365 op_cost(0);
5366 format %{ "[$reg, $off]" %}
5367 interface(MEMORY_INTER) %{
5368 base($reg);
5369 index(0xffffffff);
5370 scale(0x0);
5371 disp($off);
5372 %}
5373 %}
5374
5375 operand indOffI2(iRegP reg, immIOffset2 off)
5376 %{
5377 constraint(ALLOC_IN_RC(ptr_reg));
5378 match(AddP reg off);
5379 op_cost(0);
5380 format %{ "[$reg, $off]" %}
5381 interface(MEMORY_INTER) %{
5382 base($reg);
5383 index(0xffffffff);
5384 scale(0x0);
5385 disp($off);
5386 %}
5387 %}
5388
5389 operand indOffI4(iRegP reg, immIOffset4 off)
5390 %{
5391 constraint(ALLOC_IN_RC(ptr_reg));
5392 match(AddP reg off);
5393 op_cost(0);
5394 format %{ "[$reg, $off]" %}
5395 interface(MEMORY_INTER) %{
5396 base($reg);
5397 index(0xffffffff);
5398 scale(0x0);
5399 disp($off);
5400 %}
5401 %}
5402
5403 operand indOffI8(iRegP reg, immIOffset8 off)
5404 %{
5405 constraint(ALLOC_IN_RC(ptr_reg));
5406 match(AddP reg off);
5407 op_cost(0);
5408 format %{ "[$reg, $off]" %}
5409 interface(MEMORY_INTER) %{
5410 base($reg);
5411 index(0xffffffff);
5412 scale(0x0);
5413 disp($off);
5414 %}
5415 %}
5416
5417 operand indOffI16(iRegP reg, immIOffset16 off)
5418 %{
5419 constraint(ALLOC_IN_RC(ptr_reg));
5420 match(AddP reg off);
5421 op_cost(0);
5422 format %{ "[$reg, $off]" %}
5423 interface(MEMORY_INTER) %{
5424 base($reg);
5425 index(0xffffffff);
5426 scale(0x0);
5427 disp($off);
5428 %}
5429 %}
5430
5431 operand indOffL1(iRegP reg, immLoffset1 off)
5432 %{
5433 constraint(ALLOC_IN_RC(ptr_reg));
5434 match(AddP reg off);
5435 op_cost(0);
5436 format %{ "[$reg, $off]" %}
5437 interface(MEMORY_INTER) %{
5438 base($reg);
5439 index(0xffffffff);
5440 scale(0x0);
5441 disp($off);
5442 %}
5443 %}
5444
5445 operand indOffL2(iRegP reg, immLoffset2 off)
5446 %{
5447 constraint(ALLOC_IN_RC(ptr_reg));
5448 match(AddP reg off);
5449 op_cost(0);
5450 format %{ "[$reg, $off]" %}
5451 interface(MEMORY_INTER) %{
5452 base($reg);
5453 index(0xffffffff);
5454 scale(0x0);
5455 disp($off);
5456 %}
5457 %}
5458
5459 operand indOffL4(iRegP reg, immLoffset4 off)
5460 %{
5461 constraint(ALLOC_IN_RC(ptr_reg));
5462 match(AddP reg off);
5463 op_cost(0);
5464 format %{ "[$reg, $off]" %}
5465 interface(MEMORY_INTER) %{
5466 base($reg);
5467 index(0xffffffff);
5468 scale(0x0);
5469 disp($off);
5470 %}
5471 %}
5472
5473 operand indOffL8(iRegP reg, immLoffset8 off)
5474 %{
5475 constraint(ALLOC_IN_RC(ptr_reg));
5476 match(AddP reg off);
5477 op_cost(0);
5478 format %{ "[$reg, $off]" %}
5479 interface(MEMORY_INTER) %{
5480 base($reg);
5481 index(0xffffffff);
5482 scale(0x0);
5483 disp($off);
5484 %}
5485 %}
5486
5487 operand indOffL16(iRegP reg, immLoffset16 off)
5488 %{
5489 constraint(ALLOC_IN_RC(ptr_reg));
5490 match(AddP reg off);
5491 op_cost(0);
5492 format %{ "[$reg, $off]" %}
5493 interface(MEMORY_INTER) %{
5494 base($reg);
5495 index(0xffffffff);
5496 scale(0x0);
5497 disp($off);
5498 %}
5499 %}
5500
5501 operand indirectX2P(iRegL reg)
5502 %{
5503 constraint(ALLOC_IN_RC(ptr_reg));
5504 match(CastX2P reg);
5505 op_cost(0);
5506 format %{ "[$reg]\t# long -> ptr" %}
5507 interface(MEMORY_INTER) %{
5508 base($reg);
5509 index(0xffffffff);
5510 scale(0x0);
5511 disp(0x0);
5512 %}
5513 %}
5514
5515 operand indOffX2P(iRegL reg, immLOffset off)
5516 %{
5517 constraint(ALLOC_IN_RC(ptr_reg));
5518 match(AddP (CastX2P reg) off);
5519 op_cost(0);
5520 format %{ "[$reg, $off]\t# long -> ptr" %}
5521 interface(MEMORY_INTER) %{
5522 base($reg);
5523 index(0xffffffff);
5524 scale(0x0);
5525 disp($off);
5526 %}
5527 %}
5528
5529 operand indirectN(iRegN reg)
5530 %{
5531 predicate(CompressedOops::shift() == 0);
5532 constraint(ALLOC_IN_RC(ptr_reg));
5533 match(DecodeN reg);
5534 op_cost(0);
5535 format %{ "[$reg]\t# narrow" %}
5536 interface(MEMORY_INTER) %{
5537 base($reg);
5538 index(0xffffffff);
5539 scale(0x0);
5540 disp(0x0);
5541 %}
5542 %}
5543
5544 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5545 %{
5546 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5547 constraint(ALLOC_IN_RC(ptr_reg));
5548 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5549 op_cost(0);
5550 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5551 interface(MEMORY_INTER) %{
5552 base($reg);
5553 index($ireg);
5554 scale($scale);
5555 disp(0x0);
5556 %}
5557 %}
5558
5559 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5560 %{
5561 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5562 constraint(ALLOC_IN_RC(ptr_reg));
5563 match(AddP (DecodeN reg) (LShiftL lreg scale));
5564 op_cost(0);
5565 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5566 interface(MEMORY_INTER) %{
5567 base($reg);
5568 index($lreg);
5569 scale($scale);
5570 disp(0x0);
5571 %}
5572 %}
5573
5574 operand indIndexI2LN(iRegN reg, iRegI ireg)
5575 %{
5576 predicate(CompressedOops::shift() == 0);
5577 constraint(ALLOC_IN_RC(ptr_reg));
5578 match(AddP (DecodeN reg) (ConvI2L ireg));
5579 op_cost(0);
5580 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5581 interface(MEMORY_INTER) %{
5582 base($reg);
5583 index($ireg);
5584 scale(0x0);
5585 disp(0x0);
5586 %}
5587 %}
5588
5589 operand indIndexN(iRegN reg, iRegL lreg)
5590 %{
5591 predicate(CompressedOops::shift() == 0);
5592 constraint(ALLOC_IN_RC(ptr_reg));
5593 match(AddP (DecodeN reg) lreg);
5594 op_cost(0);
5595 format %{ "$reg, $lreg\t# narrow" %}
5596 interface(MEMORY_INTER) %{
5597 base($reg);
5598 index($lreg);
5599 scale(0x0);
5600 disp(0x0);
5601 %}
5602 %}
5603
5604 operand indOffIN(iRegN reg, immIOffset off)
5605 %{
5606 predicate(CompressedOops::shift() == 0);
5607 constraint(ALLOC_IN_RC(ptr_reg));
5608 match(AddP (DecodeN reg) off);
5609 op_cost(0);
5610 format %{ "[$reg, $off]\t# narrow" %}
5611 interface(MEMORY_INTER) %{
5612 base($reg);
5613 index(0xffffffff);
5614 scale(0x0);
5615 disp($off);
5616 %}
5617 %}
5618
5619 operand indOffLN(iRegN reg, immLOffset off)
5620 %{
5621 predicate(CompressedOops::shift() == 0);
5622 constraint(ALLOC_IN_RC(ptr_reg));
5623 match(AddP (DecodeN reg) off);
5624 op_cost(0);
5625 format %{ "[$reg, $off]\t# narrow" %}
5626 interface(MEMORY_INTER) %{
5627 base($reg);
5628 index(0xffffffff);
5629 scale(0x0);
5630 disp($off);
5631 %}
5632 %}
5633
5634
5635 //----------Special Memory Operands--------------------------------------------
5636 // Stack Slot Operand - This operand is used for loading and storing temporary
5637 // values on the stack where a match requires a value to
5638 // flow through memory.
5639 operand stackSlotP(sRegP reg)
5640 %{
5641 constraint(ALLOC_IN_RC(stack_slots));
5642 op_cost(100);
5643 // No match rule because this operand is only generated in matching
5644 // match(RegP);
5645 format %{ "[$reg]" %}
5646 interface(MEMORY_INTER) %{
5647 base(0x1e); // RSP
5648 index(0x0); // No Index
5649 scale(0x0); // No Scale
5650 disp($reg); // Stack Offset
5651 %}
5652 %}
5653
5654 operand stackSlotI(sRegI reg)
5655 %{
5656 constraint(ALLOC_IN_RC(stack_slots));
5657 // No match rule because this operand is only generated in matching
5658 // match(RegI);
5659 format %{ "[$reg]" %}
5660 interface(MEMORY_INTER) %{
5661 base(0x1e); // RSP
5662 index(0x0); // No Index
5663 scale(0x0); // No Scale
5664 disp($reg); // Stack Offset
5665 %}
5666 %}
5667
5668 operand stackSlotF(sRegF reg)
5669 %{
5670 constraint(ALLOC_IN_RC(stack_slots));
5671 // No match rule because this operand is only generated in matching
5672 // match(RegF);
5673 format %{ "[$reg]" %}
5674 interface(MEMORY_INTER) %{
5675 base(0x1e); // RSP
5676 index(0x0); // No Index
5677 scale(0x0); // No Scale
5678 disp($reg); // Stack Offset
5679 %}
5680 %}
5681
5682 operand stackSlotD(sRegD reg)
5683 %{
5684 constraint(ALLOC_IN_RC(stack_slots));
5685 // No match rule because this operand is only generated in matching
5686 // match(RegD);
5687 format %{ "[$reg]" %}
5688 interface(MEMORY_INTER) %{
5689 base(0x1e); // RSP
5690 index(0x0); // No Index
5691 scale(0x0); // No Scale
5692 disp($reg); // Stack Offset
5693 %}
5694 %}
5695
5696 operand stackSlotL(sRegL reg)
5697 %{
5698 constraint(ALLOC_IN_RC(stack_slots));
5699 // No match rule because this operand is only generated in matching
5700 // match(RegL);
5701 format %{ "[$reg]" %}
5702 interface(MEMORY_INTER) %{
5703 base(0x1e); // RSP
5704 index(0x0); // No Index
5705 scale(0x0); // No Scale
5706 disp($reg); // Stack Offset
5707 %}
5708 %}
5709
5710 // Operands for expressing Control Flow
5711 // NOTE: Label is a predefined operand which should not be redefined in
5712 // the AD file. It is generically handled within the ADLC.
5713
5714 //----------Conditional Branch Operands----------------------------------------
5715 // Comparison Op - This is the operation of the comparison, and is limited to
5716 // the following set of codes:
5717 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5718 //
5719 // Other attributes of the comparison, such as unsignedness, are specified
5720 // by the comparison instruction that sets a condition code flags register.
5721 // That result is represented by a flags operand whose subtype is appropriate
5722 // to the unsignedness (etc.) of the comparison.
5723 //
5724 // Later, the instruction which matches both the Comparison Op (a Bool) and
5725 // the flags (produced by the Cmp) specifies the coding of the comparison op
5726 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5727
5728 // used for signed integral comparisons and fp comparisons
5729
5730 operand cmpOp()
5731 %{
5732 match(Bool);
5733
5734 format %{ "" %}
5735 interface(COND_INTER) %{
5736 equal(0x0, "eq");
5737 not_equal(0x1, "ne");
5738 less(0xb, "lt");
5739 greater_equal(0xa, "ge");
5740 less_equal(0xd, "le");
5741 greater(0xc, "gt");
5742 overflow(0x6, "vs");
5743 no_overflow(0x7, "vc");
5744 %}
5745 %}
5746
5747 // used for unsigned integral comparisons
5748
5749 operand cmpOpU()
5750 %{
5751 match(Bool);
5752
5753 format %{ "" %}
5754 interface(COND_INTER) %{
5755 equal(0x0, "eq");
5756 not_equal(0x1, "ne");
5757 less(0x3, "lo");
5758 greater_equal(0x2, "hs");
5759 less_equal(0x9, "ls");
5760 greater(0x8, "hi");
5761 overflow(0x6, "vs");
5762 no_overflow(0x7, "vc");
5763 %}
5764 %}
5765
5766 // used for certain integral comparisons which can be
5767 // converted to cbxx or tbxx instructions
5768
5769 operand cmpOpEqNe()
5770 %{
5771 match(Bool);
5772 op_cost(0);
5773 predicate(n->as_Bool()->_test._test == BoolTest::ne
5774 || n->as_Bool()->_test._test == BoolTest::eq);
5775
5776 format %{ "" %}
5777 interface(COND_INTER) %{
5778 equal(0x0, "eq");
5779 not_equal(0x1, "ne");
5780 less(0xb, "lt");
5781 greater_equal(0xa, "ge");
5782 less_equal(0xd, "le");
5783 greater(0xc, "gt");
5784 overflow(0x6, "vs");
5785 no_overflow(0x7, "vc");
5786 %}
5787 %}
5788
5789 // used for certain integral comparisons which can be
5790 // converted to cbxx or tbxx instructions
5791
5792 operand cmpOpLtGe()
5793 %{
5794 match(Bool);
5795 op_cost(0);
5796
5797 predicate(n->as_Bool()->_test._test == BoolTest::lt
5798 || n->as_Bool()->_test._test == BoolTest::ge);
5799
5800 format %{ "" %}
5801 interface(COND_INTER) %{
5802 equal(0x0, "eq");
5803 not_equal(0x1, "ne");
5804 less(0xb, "lt");
5805 greater_equal(0xa, "ge");
5806 less_equal(0xd, "le");
5807 greater(0xc, "gt");
5808 overflow(0x6, "vs");
5809 no_overflow(0x7, "vc");
5810 %}
5811 %}
5812
5813 // used for certain unsigned integral comparisons which can be
5814 // converted to cbxx or tbxx instructions
5815
5816 operand cmpOpUEqNeLeGt()
5817 %{
5818 match(Bool);
5819 op_cost(0);
5820
5821 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5822 n->as_Bool()->_test._test == BoolTest::ne ||
5823 n->as_Bool()->_test._test == BoolTest::le ||
5824 n->as_Bool()->_test._test == BoolTest::gt);
5825
5826 format %{ "" %}
5827 interface(COND_INTER) %{
5828 equal(0x0, "eq");
5829 not_equal(0x1, "ne");
5830 less(0x3, "lo");
5831 greater_equal(0x2, "hs");
5832 less_equal(0x9, "ls");
5833 greater(0x8, "hi");
5834 overflow(0x6, "vs");
5835 no_overflow(0x7, "vc");
5836 %}
5837 %}
5838
5839 // Special operand allowing long args to int ops to be truncated for free
5840
5841 operand iRegL2I(iRegL reg) %{
5842
5843 op_cost(0);
5844
5845 match(ConvL2I reg);
5846
5847 format %{ "l2i($reg)" %}
5848
5849 interface(REG_INTER)
5850 %}
5851
5852 operand iRegL2P(iRegL reg) %{
5853
5854 op_cost(0);
5855
5856 match(CastX2P reg);
5857
5858 format %{ "l2p($reg)" %}
5859
5860 interface(REG_INTER)
5861 %}
5862
5863 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5864 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5865 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5866 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5867
5868 //----------OPERAND CLASSES----------------------------------------------------
5869 // Operand Classes are groups of operands that are used as to simplify
5870 // instruction definitions by not requiring the AD writer to specify
5871 // separate instructions for every form of operand when the
5872 // instruction accepts multiple operand types with the same basic
5873 // encoding and format. The classic case of this is memory operands.
5874
5875 // memory is used to define read/write location for load/store
5876 // instruction defs. we can turn a memory op into an Address
5877
5878 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5879 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5880
5881 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5882 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5883
5884 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5885 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5886
5887 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5888 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5889
5890 // All of the memory operands. For the pipeline description.
5891 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5892 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5893 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5894
5895
5896 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5897 // operations. it allows the src to be either an iRegI or a (ConvL2I
5898 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5899 // can be elided because the 32-bit instruction will just employ the
5900 // lower 32 bits anyway.
5901 //
5902 // n.b. this does not elide all L2I conversions. if the truncated
5903 // value is consumed by more than one operation then the ConvL2I
5904 // cannot be bundled into the consuming nodes so an l2i gets planted
5905 // (actually a movw $dst $src) and the downstream instructions consume
5906 // the result of the l2i as an iRegI input. That's a shame since the
5907 // movw is actually redundant but its not too costly.
5908
5909 opclass iRegIorL2I(iRegI, iRegL2I);
5910 opclass iRegPorL2P(iRegP, iRegL2P);
5911
5912 //----------PIPELINE-----------------------------------------------------------
5913 // Rules which define the behavior of the target architectures pipeline.
5914
5915 // For specific pipelines, eg A53, define the stages of that pipeline
5916 //pipe_desc(ISS, EX1, EX2, WR);
5917 #define ISS S0
5918 #define EX1 S1
5919 #define EX2 S2
5920 #define WR S3
5921
5922 // Integer ALU reg operation
5923 pipeline %{
5924
5925 attributes %{
5926 // ARM instructions are of fixed length
5927 fixed_size_instructions; // Fixed size instructions TODO does
5928 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5929 // ARM instructions come in 32-bit word units
5930 instruction_unit_size = 4; // An instruction is 4 bytes long
5931 instruction_fetch_unit_size = 64; // The processor fetches one line
5932 instruction_fetch_units = 1; // of 64 bytes
5933 %}
5934
5935 // We don't use an actual pipeline model so don't care about resources
5936 // or description. we do use pipeline classes to introduce fixed
5937 // latencies
5938
5939 //----------RESOURCES----------------------------------------------------------
5940 // Resources are the functional units available to the machine
5941
5942 resources( INS0, INS1, INS01 = INS0 | INS1,
5943 ALU0, ALU1, ALU = ALU0 | ALU1,
5944 MAC,
5945 DIV,
5946 BRANCH,
5947 LDST,
5948 NEON_FP);
5949
5950 //----------PIPELINE DESCRIPTION-----------------------------------------------
5951 // Pipeline Description specifies the stages in the machine's pipeline
5952
5953 // Define the pipeline as a generic 6 stage pipeline
5954 pipe_desc(S0, S1, S2, S3, S4, S5);
5955
5956 //----------PIPELINE CLASSES---------------------------------------------------
5957 // Pipeline Classes describe the stages in which input and output are
5958 // referenced by the hardware pipeline.
5959
5960 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5961 %{
5962 single_instruction;
5963 src1 : S1(read);
5964 src2 : S2(read);
5965 dst : S5(write);
5966 INS01 : ISS;
5967 NEON_FP : S5;
5968 %}
5969
5970 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5971 %{
5972 single_instruction;
5973 src1 : S1(read);
5974 src2 : S2(read);
5975 dst : S5(write);
5976 INS01 : ISS;
5977 NEON_FP : S5;
5978 %}
5979
5980 pipe_class fp_uop_s(vRegF dst, vRegF src)
5981 %{
5982 single_instruction;
5983 src : S1(read);
5984 dst : S5(write);
5985 INS01 : ISS;
5986 NEON_FP : S5;
5987 %}
5988
5989 pipe_class fp_uop_d(vRegD dst, vRegD src)
5990 %{
5991 single_instruction;
5992 src : S1(read);
5993 dst : S5(write);
5994 INS01 : ISS;
5995 NEON_FP : S5;
5996 %}
5997
5998 pipe_class fp_d2f(vRegF dst, vRegD src)
5999 %{
6000 single_instruction;
6001 src : S1(read);
6002 dst : S5(write);
6003 INS01 : ISS;
6004 NEON_FP : S5;
6005 %}
6006
6007 pipe_class fp_f2d(vRegD dst, vRegF src)
6008 %{
6009 single_instruction;
6010 src : S1(read);
6011 dst : S5(write);
6012 INS01 : ISS;
6013 NEON_FP : S5;
6014 %}
6015
6016 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6017 %{
6018 single_instruction;
6019 src : S1(read);
6020 dst : S5(write);
6021 INS01 : ISS;
6022 NEON_FP : S5;
6023 %}
6024
6025 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6026 %{
6027 single_instruction;
6028 src : S1(read);
6029 dst : S5(write);
6030 INS01 : ISS;
6031 NEON_FP : S5;
6032 %}
6033
6034 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6035 %{
6036 single_instruction;
6037 src : S1(read);
6038 dst : S5(write);
6039 INS01 : ISS;
6040 NEON_FP : S5;
6041 %}
6042
6043 pipe_class fp_l2f(vRegF dst, iRegL src)
6044 %{
6045 single_instruction;
6046 src : S1(read);
6047 dst : S5(write);
6048 INS01 : ISS;
6049 NEON_FP : S5;
6050 %}
6051
6052 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6053 %{
6054 single_instruction;
6055 src : S1(read);
6056 dst : S5(write);
6057 INS01 : ISS;
6058 NEON_FP : S5;
6059 %}
6060
6061 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6062 %{
6063 single_instruction;
6064 src : S1(read);
6065 dst : S5(write);
6066 INS01 : ISS;
6067 NEON_FP : S5;
6068 %}
6069
6070 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6071 %{
6072 single_instruction;
6073 src : S1(read);
6074 dst : S5(write);
6075 INS01 : ISS;
6076 NEON_FP : S5;
6077 %}
6078
6079 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6080 %{
6081 single_instruction;
6082 src : S1(read);
6083 dst : S5(write);
6084 INS01 : ISS;
6085 NEON_FP : S5;
6086 %}
6087
6088 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6089 %{
6090 single_instruction;
6091 src1 : S1(read);
6092 src2 : S2(read);
6093 dst : S5(write);
6094 INS0 : ISS;
6095 NEON_FP : S5;
6096 %}
6097
6098 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6099 %{
6100 single_instruction;
6101 src1 : S1(read);
6102 src2 : S2(read);
6103 dst : S5(write);
6104 INS0 : ISS;
6105 NEON_FP : S5;
6106 %}
6107
6108 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6109 %{
6110 single_instruction;
6111 cr : S1(read);
6112 src1 : S1(read);
6113 src2 : S1(read);
6114 dst : S3(write);
6115 INS01 : ISS;
6116 NEON_FP : S3;
6117 %}
6118
6119 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6120 %{
6121 single_instruction;
6122 cr : S1(read);
6123 src1 : S1(read);
6124 src2 : S1(read);
6125 dst : S3(write);
6126 INS01 : ISS;
6127 NEON_FP : S3;
6128 %}
6129
6130 pipe_class fp_imm_s(vRegF dst)
6131 %{
6132 single_instruction;
6133 dst : S3(write);
6134 INS01 : ISS;
6135 NEON_FP : S3;
6136 %}
6137
6138 pipe_class fp_imm_d(vRegD dst)
6139 %{
6140 single_instruction;
6141 dst : S3(write);
6142 INS01 : ISS;
6143 NEON_FP : S3;
6144 %}
6145
6146 pipe_class fp_load_constant_s(vRegF dst)
6147 %{
6148 single_instruction;
6149 dst : S4(write);
6150 INS01 : ISS;
6151 NEON_FP : S4;
6152 %}
6153
6154 pipe_class fp_load_constant_d(vRegD dst)
6155 %{
6156 single_instruction;
6157 dst : S4(write);
6158 INS01 : ISS;
6159 NEON_FP : S4;
6160 %}
6161
6162 //------- Integer ALU operations --------------------------
6163
6164 // Integer ALU reg-reg operation
6165 // Operands needed in EX1, result generated in EX2
6166 // Eg. ADD x0, x1, x2
6167 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6168 %{
6169 single_instruction;
6170 dst : EX2(write);
6171 src1 : EX1(read);
6172 src2 : EX1(read);
6173 INS01 : ISS; // Dual issue as instruction 0 or 1
6174 ALU : EX2;
6175 %}
6176
6177 // Integer ALU reg-reg operation with constant shift
6178 // Shifted register must be available in LATE_ISS instead of EX1
6179 // Eg. ADD x0, x1, x2, LSL #2
6180 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6181 %{
6182 single_instruction;
6183 dst : EX2(write);
6184 src1 : EX1(read);
6185 src2 : ISS(read);
6186 INS01 : ISS;
6187 ALU : EX2;
6188 %}
6189
6190 // Integer ALU reg operation with constant shift
6191 // Eg. LSL x0, x1, #shift
6192 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6193 %{
6194 single_instruction;
6195 dst : EX2(write);
6196 src1 : ISS(read);
6197 INS01 : ISS;
6198 ALU : EX2;
6199 %}
6200
6201 // Integer ALU reg-reg operation with variable shift
6202 // Both operands must be available in LATE_ISS instead of EX1
6203 // Result is available in EX1 instead of EX2
6204 // Eg. LSLV x0, x1, x2
6205 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6206 %{
6207 single_instruction;
6208 dst : EX1(write);
6209 src1 : ISS(read);
6210 src2 : ISS(read);
6211 INS01 : ISS;
6212 ALU : EX1;
6213 %}
6214
6215 // Integer ALU reg-reg operation with extract
6216 // As for _vshift above, but result generated in EX2
6217 // Eg. EXTR x0, x1, x2, #N
6218 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6219 %{
6220 single_instruction;
6221 dst : EX2(write);
6222 src1 : ISS(read);
6223 src2 : ISS(read);
6224 INS1 : ISS; // Can only dual issue as Instruction 1
6225 ALU : EX1;
6226 %}
6227
6228 // Integer ALU reg operation
6229 // Eg. NEG x0, x1
6230 pipe_class ialu_reg(iRegI dst, iRegI src)
6231 %{
6232 single_instruction;
6233 dst : EX2(write);
6234 src : EX1(read);
6235 INS01 : ISS;
6236 ALU : EX2;
6237 %}
6238
6239 // Integer ALU reg mmediate operation
6240 // Eg. ADD x0, x1, #N
6241 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6242 %{
6243 single_instruction;
6244 dst : EX2(write);
6245 src1 : EX1(read);
6246 INS01 : ISS;
6247 ALU : EX2;
6248 %}
6249
6250 // Integer ALU immediate operation (no source operands)
6251 // Eg. MOV x0, #N
6252 pipe_class ialu_imm(iRegI dst)
6253 %{
6254 single_instruction;
6255 dst : EX1(write);
6256 INS01 : ISS;
6257 ALU : EX1;
6258 %}
6259
6260 //------- Compare operation -------------------------------
6261
6262 // Compare reg-reg
6263 // Eg. CMP x0, x1
6264 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6265 %{
6266 single_instruction;
6267 // fixed_latency(16);
6268 cr : EX2(write);
6269 op1 : EX1(read);
6270 op2 : EX1(read);
6271 INS01 : ISS;
6272 ALU : EX2;
6273 %}
6274
6275 // Compare reg-reg
6276 // Eg. CMP x0, #N
6277 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6278 %{
6279 single_instruction;
6280 // fixed_latency(16);
6281 cr : EX2(write);
6282 op1 : EX1(read);
6283 INS01 : ISS;
6284 ALU : EX2;
6285 %}
6286
6287 //------- Conditional instructions ------------------------
6288
6289 // Conditional no operands
6290 // Eg. CSINC x0, zr, zr, <cond>
6291 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6292 %{
6293 single_instruction;
6294 cr : EX1(read);
6295 dst : EX2(write);
6296 INS01 : ISS;
6297 ALU : EX2;
6298 %}
6299
6300 // Conditional 2 operand
6301 // EG. CSEL X0, X1, X2, <cond>
6302 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6303 %{
6304 single_instruction;
6305 cr : EX1(read);
6306 src1 : EX1(read);
6307 src2 : EX1(read);
6308 dst : EX2(write);
6309 INS01 : ISS;
6310 ALU : EX2;
6311 %}
6312
6313 // Conditional 2 operand
6314 // EG. CSEL X0, X1, X2, <cond>
6315 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6316 %{
6317 single_instruction;
6318 cr : EX1(read);
6319 src : EX1(read);
6320 dst : EX2(write);
6321 INS01 : ISS;
6322 ALU : EX2;
6323 %}
6324
6325 //------- Multiply pipeline operations --------------------
6326
6327 // Multiply reg-reg
6328 // Eg. MUL w0, w1, w2
6329 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6330 %{
6331 single_instruction;
6332 dst : WR(write);
6333 src1 : ISS(read);
6334 src2 : ISS(read);
6335 INS01 : ISS;
6336 MAC : WR;
6337 %}
6338
6339 // Multiply accumulate
6340 // Eg. MADD w0, w1, w2, w3
6341 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6342 %{
6343 single_instruction;
6344 dst : WR(write);
6345 src1 : ISS(read);
6346 src2 : ISS(read);
6347 src3 : ISS(read);
6348 INS01 : ISS;
6349 MAC : WR;
6350 %}
6351
6352 // Eg. MUL w0, w1, w2
6353 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6354 %{
6355 single_instruction;
6356 fixed_latency(3); // Maximum latency for 64 bit mul
6357 dst : WR(write);
6358 src1 : ISS(read);
6359 src2 : ISS(read);
6360 INS01 : ISS;
6361 MAC : WR;
6362 %}
6363
6364 // Multiply accumulate
6365 // Eg. MADD w0, w1, w2, w3
6366 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6367 %{
6368 single_instruction;
6369 fixed_latency(3); // Maximum latency for 64 bit mul
6370 dst : WR(write);
6371 src1 : ISS(read);
6372 src2 : ISS(read);
6373 src3 : ISS(read);
6374 INS01 : ISS;
6375 MAC : WR;
6376 %}
6377
6378 //------- Divide pipeline operations --------------------
6379
6380 // Eg. SDIV w0, w1, w2
6381 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6382 %{
6383 single_instruction;
6384 fixed_latency(8); // Maximum latency for 32 bit divide
6385 dst : WR(write);
6386 src1 : ISS(read);
6387 src2 : ISS(read);
6388 INS0 : ISS; // Can only dual issue as instruction 0
6389 DIV : WR;
6390 %}
6391
6392 // Eg. SDIV x0, x1, x2
6393 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6394 %{
6395 single_instruction;
6396 fixed_latency(16); // Maximum latency for 64 bit divide
6397 dst : WR(write);
6398 src1 : ISS(read);
6399 src2 : ISS(read);
6400 INS0 : ISS; // Can only dual issue as instruction 0
6401 DIV : WR;
6402 %}
6403
6404 //------- Load pipeline operations ------------------------
6405
6406 // Load - prefetch
6407 // Eg. PFRM <mem>
6408 pipe_class iload_prefetch(memory mem)
6409 %{
6410 single_instruction;
6411 mem : ISS(read);
6412 INS01 : ISS;
6413 LDST : WR;
6414 %}
6415
6416 // Load - reg, mem
6417 // Eg. LDR x0, <mem>
6418 pipe_class iload_reg_mem(iRegI dst, memory mem)
6419 %{
6420 single_instruction;
6421 dst : WR(write);
6422 mem : ISS(read);
6423 INS01 : ISS;
6424 LDST : WR;
6425 %}
6426
6427 // Load - reg, reg
6428 // Eg. LDR x0, [sp, x1]
6429 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6430 %{
6431 single_instruction;
6432 dst : WR(write);
6433 src : ISS(read);
6434 INS01 : ISS;
6435 LDST : WR;
6436 %}
6437
6438 //------- Store pipeline operations -----------------------
6439
6440 // Store - zr, mem
6441 // Eg. STR zr, <mem>
6442 pipe_class istore_mem(memory mem)
6443 %{
6444 single_instruction;
6445 mem : ISS(read);
6446 INS01 : ISS;
6447 LDST : WR;
6448 %}
6449
6450 // Store - reg, mem
6451 // Eg. STR x0, <mem>
6452 pipe_class istore_reg_mem(iRegI src, memory mem)
6453 %{
6454 single_instruction;
6455 mem : ISS(read);
6456 src : EX2(read);
6457 INS01 : ISS;
6458 LDST : WR;
6459 %}
6460
6461 // Store - reg, reg
6462 // Eg. STR x0, [sp, x1]
6463 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6464 %{
6465 single_instruction;
6466 dst : ISS(read);
6467 src : EX2(read);
6468 INS01 : ISS;
6469 LDST : WR;
6470 %}
6471
6472 //------- Store pipeline operations -----------------------
6473
6474 // Branch
6475 pipe_class pipe_branch()
6476 %{
6477 single_instruction;
6478 INS01 : ISS;
6479 BRANCH : EX1;
6480 %}
6481
6482 // Conditional branch
6483 pipe_class pipe_branch_cond(rFlagsReg cr)
6484 %{
6485 single_instruction;
6486 cr : EX1(read);
6487 INS01 : ISS;
6488 BRANCH : EX1;
6489 %}
6490
6491 // Compare & Branch
6492 // EG. CBZ/CBNZ
6493 pipe_class pipe_cmp_branch(iRegI op1)
6494 %{
6495 single_instruction;
6496 op1 : EX1(read);
6497 INS01 : ISS;
6498 BRANCH : EX1;
6499 %}
6500
6501 //------- Synchronisation operations ----------------------
6502
6503 // Any operation requiring serialization.
6504 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6505 pipe_class pipe_serial()
6506 %{
6507 single_instruction;
6508 force_serialization;
6509 fixed_latency(16);
6510 INS01 : ISS(2); // Cannot dual issue with any other instruction
6511 LDST : WR;
6512 %}
6513
6514 // Generic big/slow expanded idiom - also serialized
6515 pipe_class pipe_slow()
6516 %{
6517 instruction_count(10);
6518 multiple_bundles;
6519 force_serialization;
6520 fixed_latency(16);
6521 INS01 : ISS(2); // Cannot dual issue with any other instruction
6522 LDST : WR;
6523 %}
6524
6525 // Empty pipeline class
6526 pipe_class pipe_class_empty()
6527 %{
6528 single_instruction;
6529 fixed_latency(0);
6530 %}
6531
6532 // Default pipeline class.
6533 pipe_class pipe_class_default()
6534 %{
6535 single_instruction;
6536 fixed_latency(2);
6537 %}
6538
6539 // Pipeline class for compares.
6540 pipe_class pipe_class_compare()
6541 %{
6542 single_instruction;
6543 fixed_latency(16);
6544 %}
6545
6546 // Pipeline class for memory operations.
6547 pipe_class pipe_class_memory()
6548 %{
6549 single_instruction;
6550 fixed_latency(16);
6551 %}
6552
6553 // Pipeline class for call.
6554 pipe_class pipe_class_call()
6555 %{
6556 single_instruction;
6557 fixed_latency(100);
6558 %}
6559
6560 // Define the class for the Nop node.
6561 define %{
6562 MachNop = pipe_class_empty;
6563 %}
6564
6565 %}
6566 //----------INSTRUCTIONS-------------------------------------------------------
6567 //
6568 // match -- States which machine-independent subtree may be replaced
6569 // by this instruction.
6570 // ins_cost -- The estimated cost of this instruction is used by instruction
6571 // selection to identify a minimum cost tree of machine
6572 // instructions that matches a tree of machine-independent
6573 // instructions.
6574 // format -- A string providing the disassembly for this instruction.
6575 // The value of an instruction's operand may be inserted
6576 // by referring to it with a '$' prefix.
6577 // opcode -- Three instruction opcodes may be provided. These are referred
6578 // to within an encode class as $primary, $secondary, and $tertiary
6579 // rrspectively. The primary opcode is commonly used to
6580 // indicate the type of machine instruction, while secondary
6581 // and tertiary are often used for prefix options or addressing
6582 // modes.
6583 // ins_encode -- A list of encode classes with parameters. The encode class
6584 // name must have been defined in an 'enc_class' specification
6585 // in the encode section of the architecture description.
6586
6587 // ============================================================================
6588 // Memory (Load/Store) Instructions
6589
6590 // Load Instructions
6591
6592 // Load Byte (8 bit signed)
6593 instruct loadB(iRegINoSp dst, memory1 mem)
6594 %{
6595 match(Set dst (LoadB mem));
6596 predicate(!needs_acquiring_load(n));
6597
6598 ins_cost(4 * INSN_COST);
6599 format %{ "ldrsbw $dst, $mem\t# byte" %}
6600
6601 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6602
6603 ins_pipe(iload_reg_mem);
6604 %}
6605
6606 // Load Byte (8 bit signed) into long
6607 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6608 %{
6609 match(Set dst (ConvI2L (LoadB mem)));
6610 predicate(!needs_acquiring_load(n->in(1)));
6611
6612 ins_cost(4 * INSN_COST);
6613 format %{ "ldrsb $dst, $mem\t# byte" %}
6614
6615 ins_encode(aarch64_enc_ldrsb(dst, mem));
6616
6617 ins_pipe(iload_reg_mem);
6618 %}
6619
6620 // Load Byte (8 bit unsigned)
6621 instruct loadUB(iRegINoSp dst, memory1 mem)
6622 %{
6623 match(Set dst (LoadUB mem));
6624 predicate(!needs_acquiring_load(n));
6625
6626 ins_cost(4 * INSN_COST);
6627 format %{ "ldrbw $dst, $mem\t# byte" %}
6628
6629 ins_encode(aarch64_enc_ldrb(dst, mem));
6630
6631 ins_pipe(iload_reg_mem);
6632 %}
6633
6634 // Load Byte (8 bit unsigned) into long
6635 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6636 %{
6637 match(Set dst (ConvI2L (LoadUB mem)));
6638 predicate(!needs_acquiring_load(n->in(1)));
6639
6640 ins_cost(4 * INSN_COST);
6641 format %{ "ldrb $dst, $mem\t# byte" %}
6642
6643 ins_encode(aarch64_enc_ldrb(dst, mem));
6644
6645 ins_pipe(iload_reg_mem);
6646 %}
6647
6648 // Load Short (16 bit signed)
6649 instruct loadS(iRegINoSp dst, memory2 mem)
6650 %{
6651 match(Set dst (LoadS mem));
6652 predicate(!needs_acquiring_load(n));
6653
6654 ins_cost(4 * INSN_COST);
6655 format %{ "ldrshw $dst, $mem\t# short" %}
6656
6657 ins_encode(aarch64_enc_ldrshw(dst, mem));
6658
6659 ins_pipe(iload_reg_mem);
6660 %}
6661
6662 // Load Short (16 bit signed) into long
6663 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6664 %{
6665 match(Set dst (ConvI2L (LoadS mem)));
6666 predicate(!needs_acquiring_load(n->in(1)));
6667
6668 ins_cost(4 * INSN_COST);
6669 format %{ "ldrsh $dst, $mem\t# short" %}
6670
6671 ins_encode(aarch64_enc_ldrsh(dst, mem));
6672
6673 ins_pipe(iload_reg_mem);
6674 %}
6675
6676 // Load Char (16 bit unsigned)
6677 instruct loadUS(iRegINoSp dst, memory2 mem)
6678 %{
6679 match(Set dst (LoadUS mem));
6680 predicate(!needs_acquiring_load(n));
6681
6682 ins_cost(4 * INSN_COST);
6683 format %{ "ldrh $dst, $mem\t# short" %}
6684
6685 ins_encode(aarch64_enc_ldrh(dst, mem));
6686
6687 ins_pipe(iload_reg_mem);
6688 %}
6689
6690 // Load Short/Char (16 bit unsigned) into long
6691 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6692 %{
6693 match(Set dst (ConvI2L (LoadUS mem)));
6694 predicate(!needs_acquiring_load(n->in(1)));
6695
6696 ins_cost(4 * INSN_COST);
6697 format %{ "ldrh $dst, $mem\t# short" %}
6698
6699 ins_encode(aarch64_enc_ldrh(dst, mem));
6700
6701 ins_pipe(iload_reg_mem);
6702 %}
6703
6704 // Load Integer (32 bit signed)
6705 instruct loadI(iRegINoSp dst, memory4 mem)
6706 %{
6707 match(Set dst (LoadI mem));
6708 predicate(!needs_acquiring_load(n));
6709
6710 ins_cost(4 * INSN_COST);
6711 format %{ "ldrw $dst, $mem\t# int" %}
6712
6713 ins_encode(aarch64_enc_ldrw(dst, mem));
6714
6715 ins_pipe(iload_reg_mem);
6716 %}
6717
6718 // Load Integer (32 bit signed) into long
6719 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6720 %{
6721 match(Set dst (ConvI2L (LoadI mem)));
6722 predicate(!needs_acquiring_load(n->in(1)));
6723
6724 ins_cost(4 * INSN_COST);
6725 format %{ "ldrsw $dst, $mem\t# int" %}
6726
6727 ins_encode(aarch64_enc_ldrsw(dst, mem));
6728
6729 ins_pipe(iload_reg_mem);
6730 %}
6731
6732 // Load Integer (32 bit unsigned) into long
6733 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6734 %{
6735 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6736 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6737
6738 ins_cost(4 * INSN_COST);
6739 format %{ "ldrw $dst, $mem\t# int" %}
6740
6741 ins_encode(aarch64_enc_ldrw(dst, mem));
6742
6743 ins_pipe(iload_reg_mem);
6744 %}
6745
6746 // Load Long (64 bit signed)
6747 instruct loadL(iRegLNoSp dst, memory8 mem)
6748 %{
6749 match(Set dst (LoadL mem));
6750 predicate(!needs_acquiring_load(n));
6751
6752 ins_cost(4 * INSN_COST);
6753 format %{ "ldr $dst, $mem\t# int" %}
6754
6755 ins_encode(aarch64_enc_ldr(dst, mem));
6756
6757 ins_pipe(iload_reg_mem);
6758 %}
6759
6760 // Load Range
6761 instruct loadRange(iRegINoSp dst, memory4 mem)
6762 %{
6763 match(Set dst (LoadRange mem));
6764
6765 ins_cost(4 * INSN_COST);
6766 format %{ "ldrw $dst, $mem\t# range" %}
6767
6768 ins_encode(aarch64_enc_ldrw(dst, mem));
6769
6770 ins_pipe(iload_reg_mem);
6771 %}
6772
6773 // Load Pointer
6774 instruct loadP(iRegPNoSp dst, memory8 mem)
6775 %{
6776 match(Set dst (LoadP mem));
6777 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6778
6779 ins_cost(4 * INSN_COST);
6780 format %{ "ldr $dst, $mem\t# ptr" %}
6781
6782 ins_encode(aarch64_enc_ldr(dst, mem));
6783
6784 ins_pipe(iload_reg_mem);
6785 %}
6786
6787 // Load Compressed Pointer
6788 instruct loadN(iRegNNoSp dst, memory4 mem)
6789 %{
6790 match(Set dst (LoadN mem));
6791 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6792
6793 ins_cost(4 * INSN_COST);
6794 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6795
6796 ins_encode(aarch64_enc_ldrw(dst, mem));
6797
6798 ins_pipe(iload_reg_mem);
6799 %}
6800
6801 // Load Klass Pointer
6802 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6803 %{
6804 match(Set dst (LoadKlass mem));
6805 predicate(!needs_acquiring_load(n));
6806
6807 ins_cost(4 * INSN_COST);
6808 format %{ "ldr $dst, $mem\t# class" %}
6809
6810 ins_encode(aarch64_enc_ldr(dst, mem));
6811
6812 ins_pipe(iload_reg_mem);
6813 %}
6814
6815 // Load Narrow Klass Pointer
6816 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6817 %{
6818 match(Set dst (LoadNKlass mem));
6819 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6820
6821 ins_cost(4 * INSN_COST);
6822 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6823
6824 ins_encode(aarch64_enc_ldrw(dst, mem));
6825
6826 ins_pipe(iload_reg_mem);
6827 %}
6828
6829 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6830 %{
6831 match(Set dst (LoadNKlass mem));
6832 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6833
6834 ins_cost(4 * INSN_COST);
6835 format %{
6836 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6837 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6838 %}
6839 ins_encode %{
6840 // inlined aarch64_enc_ldrw
6841 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6842 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6843 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6844 %}
6845 ins_pipe(iload_reg_mem);
6846 %}
6847
6848 // Load Float
6849 instruct loadF(vRegF dst, memory4 mem)
6850 %{
6851 match(Set dst (LoadF mem));
6852 predicate(!needs_acquiring_load(n));
6853
6854 ins_cost(4 * INSN_COST);
6855 format %{ "ldrs $dst, $mem\t# float" %}
6856
6857 ins_encode( aarch64_enc_ldrs(dst, mem) );
6858
6859 ins_pipe(pipe_class_memory);
6860 %}
6861
6862 // Load Double
6863 instruct loadD(vRegD dst, memory8 mem)
6864 %{
6865 match(Set dst (LoadD mem));
6866 predicate(!needs_acquiring_load(n));
6867
6868 ins_cost(4 * INSN_COST);
6869 format %{ "ldrd $dst, $mem\t# double" %}
6870
6871 ins_encode( aarch64_enc_ldrd(dst, mem) );
6872
6873 ins_pipe(pipe_class_memory);
6874 %}
6875
6876
6877 // Load Int Constant
6878 instruct loadConI(iRegINoSp dst, immI src)
6879 %{
6880 match(Set dst src);
6881
6882 ins_cost(INSN_COST);
6883 format %{ "mov $dst, $src\t# int" %}
6884
6885 ins_encode( aarch64_enc_movw_imm(dst, src) );
6886
6887 ins_pipe(ialu_imm);
6888 %}
6889
6890 // Load Long Constant
6891 instruct loadConL(iRegLNoSp dst, immL src)
6892 %{
6893 match(Set dst src);
6894
6895 ins_cost(INSN_COST);
6896 format %{ "mov $dst, $src\t# long" %}
6897
6898 ins_encode( aarch64_enc_mov_imm(dst, src) );
6899
6900 ins_pipe(ialu_imm);
6901 %}
6902
6903 // Load Pointer Constant
6904
6905 instruct loadConP(iRegPNoSp dst, immP con)
6906 %{
6907 match(Set dst con);
6908
6909 ins_cost(INSN_COST * 4);
6910 format %{
6911 "mov $dst, $con\t# ptr\n\t"
6912 %}
6913
6914 ins_encode(aarch64_enc_mov_p(dst, con));
6915
6916 ins_pipe(ialu_imm);
6917 %}
6918
6919 // Load Null Pointer Constant
6920
6921 instruct loadConP0(iRegPNoSp dst, immP0 con)
6922 %{
6923 match(Set dst con);
6924
6925 ins_cost(INSN_COST);
6926 format %{ "mov $dst, $con\t# nullptr ptr" %}
6927
6928 ins_encode(aarch64_enc_mov_p0(dst, con));
6929
6930 ins_pipe(ialu_imm);
6931 %}
6932
6933 // Load Pointer Constant One
6934
6935 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6936 %{
6937 match(Set dst con);
6938
6939 ins_cost(INSN_COST);
6940 format %{ "mov $dst, $con\t# nullptr ptr" %}
6941
6942 ins_encode(aarch64_enc_mov_p1(dst, con));
6943
6944 ins_pipe(ialu_imm);
6945 %}
6946
6947 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6948 %{
6949 match(Set dst con);
6950
6951 ins_cost(INSN_COST);
6952 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6953
6954 ins_encode %{
6955 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6956 %}
6957
6958 ins_pipe(ialu_imm);
6959 %}
6960
6961 // Load Narrow Pointer Constant
6962
6963 instruct loadConN(iRegNNoSp dst, immN con)
6964 %{
6965 match(Set dst con);
6966
6967 ins_cost(INSN_COST * 4);
6968 format %{ "mov $dst, $con\t# compressed ptr" %}
6969
6970 ins_encode(aarch64_enc_mov_n(dst, con));
6971
6972 ins_pipe(ialu_imm);
6973 %}
6974
6975 // Load Narrow Null Pointer Constant
6976
6977 instruct loadConN0(iRegNNoSp dst, immN0 con)
6978 %{
6979 match(Set dst con);
6980
6981 ins_cost(INSN_COST);
6982 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6983
6984 ins_encode(aarch64_enc_mov_n0(dst, con));
6985
6986 ins_pipe(ialu_imm);
6987 %}
6988
6989 // Load Narrow Klass Constant
6990
6991 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6992 %{
6993 match(Set dst con);
6994
6995 ins_cost(INSN_COST);
6996 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6997
6998 ins_encode(aarch64_enc_mov_nk(dst, con));
6999
7000 ins_pipe(ialu_imm);
7001 %}
7002
7003 // Load Packed Float Constant
7004
7005 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7006 match(Set dst con);
7007 ins_cost(INSN_COST * 4);
7008 format %{ "fmovs $dst, $con"%}
7009 ins_encode %{
7010 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7011 %}
7012
7013 ins_pipe(fp_imm_s);
7014 %}
7015
7016 // Load Float Constant
7017
7018 instruct loadConF(vRegF dst, immF con) %{
7019 match(Set dst con);
7020
7021 ins_cost(INSN_COST * 4);
7022
7023 format %{
7024 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7025 %}
7026
7027 ins_encode %{
7028 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7029 %}
7030
7031 ins_pipe(fp_load_constant_s);
7032 %}
7033
7034 // Load Packed Double Constant
7035
7036 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7037 match(Set dst con);
7038 ins_cost(INSN_COST);
7039 format %{ "fmovd $dst, $con"%}
7040 ins_encode %{
7041 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7042 %}
7043
7044 ins_pipe(fp_imm_d);
7045 %}
7046
7047 // Load Double Constant
7048
7049 instruct loadConD(vRegD dst, immD con) %{
7050 match(Set dst con);
7051
7052 ins_cost(INSN_COST * 5);
7053 format %{
7054 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7055 %}
7056
7057 ins_encode %{
7058 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7059 %}
7060
7061 ins_pipe(fp_load_constant_d);
7062 %}
7063
7064 // Load Half Float Constant
7065 instruct loadConH(vRegF dst, immH con) %{
7066 match(Set dst con);
7067 format %{ "mov rscratch1, $con\n\t"
7068 "fmov $dst, rscratch1"
7069 %}
7070 ins_encode %{
7071 __ movw(rscratch1, (uint32_t)$con$$constant);
7072 __ fmovs($dst$$FloatRegister, rscratch1);
7073 %}
7074 ins_pipe(pipe_class_default);
7075 %}
7076
7077 // Store Instructions
7078
7079 // Store Byte
7080 instruct storeB(iRegIorL2I src, memory1 mem)
7081 %{
7082 match(Set mem (StoreB mem src));
7083 predicate(!needs_releasing_store(n));
7084
7085 ins_cost(INSN_COST);
7086 format %{ "strb $src, $mem\t# byte" %}
7087
7088 ins_encode(aarch64_enc_strb(src, mem));
7089
7090 ins_pipe(istore_reg_mem);
7091 %}
7092
7093
7094 instruct storeimmB0(immI0 zero, memory1 mem)
7095 %{
7096 match(Set mem (StoreB mem zero));
7097 predicate(!needs_releasing_store(n));
7098
7099 ins_cost(INSN_COST);
7100 format %{ "strb rscractch2, $mem\t# byte" %}
7101
7102 ins_encode(aarch64_enc_strb0(mem));
7103
7104 ins_pipe(istore_mem);
7105 %}
7106
7107 // Store Char/Short
7108 instruct storeC(iRegIorL2I src, memory2 mem)
7109 %{
7110 match(Set mem (StoreC mem src));
7111 predicate(!needs_releasing_store(n));
7112
7113 ins_cost(INSN_COST);
7114 format %{ "strh $src, $mem\t# short" %}
7115
7116 ins_encode(aarch64_enc_strh(src, mem));
7117
7118 ins_pipe(istore_reg_mem);
7119 %}
7120
7121 instruct storeimmC0(immI0 zero, memory2 mem)
7122 %{
7123 match(Set mem (StoreC mem zero));
7124 predicate(!needs_releasing_store(n));
7125
7126 ins_cost(INSN_COST);
7127 format %{ "strh zr, $mem\t# short" %}
7128
7129 ins_encode(aarch64_enc_strh0(mem));
7130
7131 ins_pipe(istore_mem);
7132 %}
7133
7134 // Store Integer
7135
7136 instruct storeI(iRegIorL2I src, memory4 mem)
7137 %{
7138 match(Set mem(StoreI mem src));
7139 predicate(!needs_releasing_store(n));
7140
7141 ins_cost(INSN_COST);
7142 format %{ "strw $src, $mem\t# int" %}
7143
7144 ins_encode(aarch64_enc_strw(src, mem));
7145
7146 ins_pipe(istore_reg_mem);
7147 %}
7148
7149 instruct storeimmI0(immI0 zero, memory4 mem)
7150 %{
7151 match(Set mem(StoreI mem zero));
7152 predicate(!needs_releasing_store(n));
7153
7154 ins_cost(INSN_COST);
7155 format %{ "strw zr, $mem\t# int" %}
7156
7157 ins_encode(aarch64_enc_strw0(mem));
7158
7159 ins_pipe(istore_mem);
7160 %}
7161
7162 // Store Long (64 bit signed)
7163 instruct storeL(iRegL src, memory8 mem)
7164 %{
7165 match(Set mem (StoreL mem src));
7166 predicate(!needs_releasing_store(n));
7167
7168 ins_cost(INSN_COST);
7169 format %{ "str $src, $mem\t# int" %}
7170
7171 ins_encode(aarch64_enc_str(src, mem));
7172
7173 ins_pipe(istore_reg_mem);
7174 %}
7175
7176 // Store Long (64 bit signed)
7177 instruct storeimmL0(immL0 zero, memory8 mem)
7178 %{
7179 match(Set mem (StoreL mem zero));
7180 predicate(!needs_releasing_store(n));
7181
7182 ins_cost(INSN_COST);
7183 format %{ "str zr, $mem\t# int" %}
7184
7185 ins_encode(aarch64_enc_str0(mem));
7186
7187 ins_pipe(istore_mem);
7188 %}
7189
7190 // Store Pointer
7191 instruct storeP(iRegP src, memory8 mem)
7192 %{
7193 match(Set mem (StoreP mem src));
7194 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7195
7196 ins_cost(INSN_COST);
7197 format %{ "str $src, $mem\t# ptr" %}
7198
7199 ins_encode(aarch64_enc_str(src, mem));
7200
7201 ins_pipe(istore_reg_mem);
7202 %}
7203
7204 // Store Pointer
7205 instruct storeimmP0(immP0 zero, memory8 mem)
7206 %{
7207 match(Set mem (StoreP mem zero));
7208 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7209
7210 ins_cost(INSN_COST);
7211 format %{ "str zr, $mem\t# ptr" %}
7212
7213 ins_encode(aarch64_enc_str0(mem));
7214
7215 ins_pipe(istore_mem);
7216 %}
7217
7218 // Store Compressed Pointer
7219 instruct storeN(iRegN src, memory4 mem)
7220 %{
7221 match(Set mem (StoreN mem src));
7222 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7223
7224 ins_cost(INSN_COST);
7225 format %{ "strw $src, $mem\t# compressed ptr" %}
7226
7227 ins_encode(aarch64_enc_strw(src, mem));
7228
7229 ins_pipe(istore_reg_mem);
7230 %}
7231
7232 instruct storeImmN0(immN0 zero, memory4 mem)
7233 %{
7234 match(Set mem (StoreN mem zero));
7235 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7236
7237 ins_cost(INSN_COST);
7238 format %{ "strw zr, $mem\t# compressed ptr" %}
7239
7240 ins_encode(aarch64_enc_strw0(mem));
7241
7242 ins_pipe(istore_mem);
7243 %}
7244
7245 // Store Float
7246 instruct storeF(vRegF src, memory4 mem)
7247 %{
7248 match(Set mem (StoreF mem src));
7249 predicate(!needs_releasing_store(n));
7250
7251 ins_cost(INSN_COST);
7252 format %{ "strs $src, $mem\t# float" %}
7253
7254 ins_encode( aarch64_enc_strs(src, mem) );
7255
7256 ins_pipe(pipe_class_memory);
7257 %}
7258
7259 // TODO
7260 // implement storeImmF0 and storeFImmPacked
7261
7262 // Store Double
7263 instruct storeD(vRegD src, memory8 mem)
7264 %{
7265 match(Set mem (StoreD mem src));
7266 predicate(!needs_releasing_store(n));
7267
7268 ins_cost(INSN_COST);
7269 format %{ "strd $src, $mem\t# double" %}
7270
7271 ins_encode( aarch64_enc_strd(src, mem) );
7272
7273 ins_pipe(pipe_class_memory);
7274 %}
7275
7276 // Store Compressed Klass Pointer
7277 instruct storeNKlass(iRegN src, memory4 mem)
7278 %{
7279 predicate(!needs_releasing_store(n));
7280 match(Set mem (StoreNKlass mem src));
7281
7282 ins_cost(INSN_COST);
7283 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7284
7285 ins_encode(aarch64_enc_strw(src, mem));
7286
7287 ins_pipe(istore_reg_mem);
7288 %}
7289
7290 // TODO
7291 // implement storeImmD0 and storeDImmPacked
7292
7293 // prefetch instructions
7294 // Must be safe to execute with invalid address (cannot fault).
7295
7296 instruct prefetchalloc( memory8 mem ) %{
7297 match(PrefetchAllocation mem);
7298
7299 ins_cost(INSN_COST);
7300 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7301
7302 ins_encode( aarch64_enc_prefetchw(mem) );
7303
7304 ins_pipe(iload_prefetch);
7305 %}
7306
7307 // ---------------- volatile loads and stores ----------------
7308
7309 // Load Byte (8 bit signed)
7310 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7311 %{
7312 match(Set dst (LoadB mem));
7313
7314 ins_cost(VOLATILE_REF_COST);
7315 format %{ "ldarsb $dst, $mem\t# byte" %}
7316
7317 ins_encode(aarch64_enc_ldarsb(dst, mem));
7318
7319 ins_pipe(pipe_serial);
7320 %}
7321
7322 // Load Byte (8 bit signed) into long
7323 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7324 %{
7325 match(Set dst (ConvI2L (LoadB mem)));
7326
7327 ins_cost(VOLATILE_REF_COST);
7328 format %{ "ldarsb $dst, $mem\t# byte" %}
7329
7330 ins_encode(aarch64_enc_ldarsb(dst, mem));
7331
7332 ins_pipe(pipe_serial);
7333 %}
7334
7335 // Load Byte (8 bit unsigned)
7336 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7337 %{
7338 match(Set dst (LoadUB mem));
7339
7340 ins_cost(VOLATILE_REF_COST);
7341 format %{ "ldarb $dst, $mem\t# byte" %}
7342
7343 ins_encode(aarch64_enc_ldarb(dst, mem));
7344
7345 ins_pipe(pipe_serial);
7346 %}
7347
7348 // Load Byte (8 bit unsigned) into long
7349 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7350 %{
7351 match(Set dst (ConvI2L (LoadUB mem)));
7352
7353 ins_cost(VOLATILE_REF_COST);
7354 format %{ "ldarb $dst, $mem\t# byte" %}
7355
7356 ins_encode(aarch64_enc_ldarb(dst, mem));
7357
7358 ins_pipe(pipe_serial);
7359 %}
7360
7361 // Load Short (16 bit signed)
7362 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7363 %{
7364 match(Set dst (LoadS mem));
7365
7366 ins_cost(VOLATILE_REF_COST);
7367 format %{ "ldarshw $dst, $mem\t# short" %}
7368
7369 ins_encode(aarch64_enc_ldarshw(dst, mem));
7370
7371 ins_pipe(pipe_serial);
7372 %}
7373
7374 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7375 %{
7376 match(Set dst (LoadUS mem));
7377
7378 ins_cost(VOLATILE_REF_COST);
7379 format %{ "ldarhw $dst, $mem\t# short" %}
7380
7381 ins_encode(aarch64_enc_ldarhw(dst, mem));
7382
7383 ins_pipe(pipe_serial);
7384 %}
7385
7386 // Load Short/Char (16 bit unsigned) into long
7387 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7388 %{
7389 match(Set dst (ConvI2L (LoadUS mem)));
7390
7391 ins_cost(VOLATILE_REF_COST);
7392 format %{ "ldarh $dst, $mem\t# short" %}
7393
7394 ins_encode(aarch64_enc_ldarh(dst, mem));
7395
7396 ins_pipe(pipe_serial);
7397 %}
7398
7399 // Load Short/Char (16 bit signed) into long
7400 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7401 %{
7402 match(Set dst (ConvI2L (LoadS mem)));
7403
7404 ins_cost(VOLATILE_REF_COST);
7405 format %{ "ldarh $dst, $mem\t# short" %}
7406
7407 ins_encode(aarch64_enc_ldarsh(dst, mem));
7408
7409 ins_pipe(pipe_serial);
7410 %}
7411
7412 // Load Integer (32 bit signed)
7413 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7414 %{
7415 match(Set dst (LoadI mem));
7416
7417 ins_cost(VOLATILE_REF_COST);
7418 format %{ "ldarw $dst, $mem\t# int" %}
7419
7420 ins_encode(aarch64_enc_ldarw(dst, mem));
7421
7422 ins_pipe(pipe_serial);
7423 %}
7424
7425 // Load Integer (32 bit unsigned) into long
7426 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7427 %{
7428 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7429
7430 ins_cost(VOLATILE_REF_COST);
7431 format %{ "ldarw $dst, $mem\t# int" %}
7432
7433 ins_encode(aarch64_enc_ldarw(dst, mem));
7434
7435 ins_pipe(pipe_serial);
7436 %}
7437
7438 // Load Long (64 bit signed)
7439 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7440 %{
7441 match(Set dst (LoadL mem));
7442
7443 ins_cost(VOLATILE_REF_COST);
7444 format %{ "ldar $dst, $mem\t# int" %}
7445
7446 ins_encode(aarch64_enc_ldar(dst, mem));
7447
7448 ins_pipe(pipe_serial);
7449 %}
7450
7451 // Load Pointer
7452 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7453 %{
7454 match(Set dst (LoadP mem));
7455 predicate(n->as_Load()->barrier_data() == 0);
7456
7457 ins_cost(VOLATILE_REF_COST);
7458 format %{ "ldar $dst, $mem\t# ptr" %}
7459
7460 ins_encode(aarch64_enc_ldar(dst, mem));
7461
7462 ins_pipe(pipe_serial);
7463 %}
7464
7465 // Load Compressed Pointer
7466 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7467 %{
7468 match(Set dst (LoadN mem));
7469 predicate(n->as_Load()->barrier_data() == 0);
7470
7471 ins_cost(VOLATILE_REF_COST);
7472 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7473
7474 ins_encode(aarch64_enc_ldarw(dst, mem));
7475
7476 ins_pipe(pipe_serial);
7477 %}
7478
7479 // Load Float
7480 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7481 %{
7482 match(Set dst (LoadF mem));
7483
7484 ins_cost(VOLATILE_REF_COST);
7485 format %{ "ldars $dst, $mem\t# float" %}
7486
7487 ins_encode( aarch64_enc_fldars(dst, mem) );
7488
7489 ins_pipe(pipe_serial);
7490 %}
7491
7492 // Load Double
7493 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7494 %{
7495 match(Set dst (LoadD mem));
7496
7497 ins_cost(VOLATILE_REF_COST);
7498 format %{ "ldard $dst, $mem\t# double" %}
7499
7500 ins_encode( aarch64_enc_fldard(dst, mem) );
7501
7502 ins_pipe(pipe_serial);
7503 %}
7504
7505 // Store Byte
7506 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7507 %{
7508 match(Set mem (StoreB mem src));
7509
7510 ins_cost(VOLATILE_REF_COST);
7511 format %{ "stlrb $src, $mem\t# byte" %}
7512
7513 ins_encode(aarch64_enc_stlrb(src, mem));
7514
7515 ins_pipe(pipe_class_memory);
7516 %}
7517
7518 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7519 %{
7520 match(Set mem (StoreB mem zero));
7521
7522 ins_cost(VOLATILE_REF_COST);
7523 format %{ "stlrb zr, $mem\t# byte" %}
7524
7525 ins_encode(aarch64_enc_stlrb0(mem));
7526
7527 ins_pipe(pipe_class_memory);
7528 %}
7529
7530 // Store Char/Short
7531 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7532 %{
7533 match(Set mem (StoreC mem src));
7534
7535 ins_cost(VOLATILE_REF_COST);
7536 format %{ "stlrh $src, $mem\t# short" %}
7537
7538 ins_encode(aarch64_enc_stlrh(src, mem));
7539
7540 ins_pipe(pipe_class_memory);
7541 %}
7542
7543 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7544 %{
7545 match(Set mem (StoreC mem zero));
7546
7547 ins_cost(VOLATILE_REF_COST);
7548 format %{ "stlrh zr, $mem\t# short" %}
7549
7550 ins_encode(aarch64_enc_stlrh0(mem));
7551
7552 ins_pipe(pipe_class_memory);
7553 %}
7554
7555 // Store Integer
7556
7557 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7558 %{
7559 match(Set mem(StoreI mem src));
7560
7561 ins_cost(VOLATILE_REF_COST);
7562 format %{ "stlrw $src, $mem\t# int" %}
7563
7564 ins_encode(aarch64_enc_stlrw(src, mem));
7565
7566 ins_pipe(pipe_class_memory);
7567 %}
7568
7569 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7570 %{
7571 match(Set mem(StoreI mem zero));
7572
7573 ins_cost(VOLATILE_REF_COST);
7574 format %{ "stlrw zr, $mem\t# int" %}
7575
7576 ins_encode(aarch64_enc_stlrw0(mem));
7577
7578 ins_pipe(pipe_class_memory);
7579 %}
7580
7581 // Store Long (64 bit signed)
7582 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7583 %{
7584 match(Set mem (StoreL mem src));
7585
7586 ins_cost(VOLATILE_REF_COST);
7587 format %{ "stlr $src, $mem\t# int" %}
7588
7589 ins_encode(aarch64_enc_stlr(src, mem));
7590
7591 ins_pipe(pipe_class_memory);
7592 %}
7593
7594 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7595 %{
7596 match(Set mem (StoreL mem zero));
7597
7598 ins_cost(VOLATILE_REF_COST);
7599 format %{ "stlr zr, $mem\t# int" %}
7600
7601 ins_encode(aarch64_enc_stlr0(mem));
7602
7603 ins_pipe(pipe_class_memory);
7604 %}
7605
7606 // Store Pointer
7607 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7608 %{
7609 match(Set mem (StoreP mem src));
7610 predicate(n->as_Store()->barrier_data() == 0);
7611
7612 ins_cost(VOLATILE_REF_COST);
7613 format %{ "stlr $src, $mem\t# ptr" %}
7614
7615 ins_encode(aarch64_enc_stlr(src, mem));
7616
7617 ins_pipe(pipe_class_memory);
7618 %}
7619
7620 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7621 %{
7622 match(Set mem (StoreP mem zero));
7623 predicate(n->as_Store()->barrier_data() == 0);
7624
7625 ins_cost(VOLATILE_REF_COST);
7626 format %{ "stlr zr, $mem\t# ptr" %}
7627
7628 ins_encode(aarch64_enc_stlr0(mem));
7629
7630 ins_pipe(pipe_class_memory);
7631 %}
7632
7633 // Store Compressed Pointer
7634 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7635 %{
7636 match(Set mem (StoreN mem src));
7637 predicate(n->as_Store()->barrier_data() == 0);
7638
7639 ins_cost(VOLATILE_REF_COST);
7640 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7641
7642 ins_encode(aarch64_enc_stlrw(src, mem));
7643
7644 ins_pipe(pipe_class_memory);
7645 %}
7646
7647 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7648 %{
7649 match(Set mem (StoreN mem zero));
7650 predicate(n->as_Store()->barrier_data() == 0);
7651
7652 ins_cost(VOLATILE_REF_COST);
7653 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7654
7655 ins_encode(aarch64_enc_stlrw0(mem));
7656
7657 ins_pipe(pipe_class_memory);
7658 %}
7659
7660 // Store Float
7661 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7662 %{
7663 match(Set mem (StoreF mem src));
7664
7665 ins_cost(VOLATILE_REF_COST);
7666 format %{ "stlrs $src, $mem\t# float" %}
7667
7668 ins_encode( aarch64_enc_fstlrs(src, mem) );
7669
7670 ins_pipe(pipe_class_memory);
7671 %}
7672
7673 // TODO
7674 // implement storeImmF0 and storeFImmPacked
7675
7676 // Store Double
7677 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7678 %{
7679 match(Set mem (StoreD mem src));
7680
7681 ins_cost(VOLATILE_REF_COST);
7682 format %{ "stlrd $src, $mem\t# double" %}
7683
7684 ins_encode( aarch64_enc_fstlrd(src, mem) );
7685
7686 ins_pipe(pipe_class_memory);
7687 %}
7688
7689 // ---------------- end of volatile loads and stores ----------------
7690
7691 instruct cacheWB(indirect addr)
7692 %{
7693 predicate(VM_Version::supports_data_cache_line_flush());
7694 match(CacheWB addr);
7695
7696 ins_cost(100);
7697 format %{"cache wb $addr" %}
7698 ins_encode %{
7699 assert($addr->index_position() < 0, "should be");
7700 assert($addr$$disp == 0, "should be");
7701 __ cache_wb(Address($addr$$base$$Register, 0));
7702 %}
7703 ins_pipe(pipe_slow); // XXX
7704 %}
7705
7706 instruct cacheWBPreSync()
7707 %{
7708 predicate(VM_Version::supports_data_cache_line_flush());
7709 match(CacheWBPreSync);
7710
7711 ins_cost(100);
7712 format %{"cache wb presync" %}
7713 ins_encode %{
7714 __ cache_wbsync(true);
7715 %}
7716 ins_pipe(pipe_slow); // XXX
7717 %}
7718
7719 instruct cacheWBPostSync()
7720 %{
7721 predicate(VM_Version::supports_data_cache_line_flush());
7722 match(CacheWBPostSync);
7723
7724 ins_cost(100);
7725 format %{"cache wb postsync" %}
7726 ins_encode %{
7727 __ cache_wbsync(false);
7728 %}
7729 ins_pipe(pipe_slow); // XXX
7730 %}
7731
7732 // ============================================================================
7733 // BSWAP Instructions
7734
7735 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7736 match(Set dst (ReverseBytesI src));
7737
7738 ins_cost(INSN_COST);
7739 format %{ "revw $dst, $src" %}
7740
7741 ins_encode %{
7742 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7743 %}
7744
7745 ins_pipe(ialu_reg);
7746 %}
7747
7748 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7749 match(Set dst (ReverseBytesL src));
7750
7751 ins_cost(INSN_COST);
7752 format %{ "rev $dst, $src" %}
7753
7754 ins_encode %{
7755 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7756 %}
7757
7758 ins_pipe(ialu_reg);
7759 %}
7760
7761 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7762 match(Set dst (ReverseBytesUS src));
7763
7764 ins_cost(INSN_COST);
7765 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7766
7767 ins_encode %{
7768 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7769 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7770 %}
7771
7772 ins_pipe(ialu_reg);
7773 %}
7774
7775 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7776 match(Set dst (ReverseBytesS src));
7777
7778 ins_cost(INSN_COST);
7779 format %{ "rev16w $dst, $src\n\t"
7780 "sbfmw $dst, $dst, #0, #15" %}
7781
7782 ins_encode %{
7783 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7784 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7785 %}
7786
7787 ins_pipe(ialu_reg);
7788 %}
7789
7790 // ============================================================================
7791 // Zero Count Instructions
7792
7793 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7794 match(Set dst (CountLeadingZerosI src));
7795
7796 ins_cost(INSN_COST);
7797 format %{ "clzw $dst, $src" %}
7798 ins_encode %{
7799 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7800 %}
7801
7802 ins_pipe(ialu_reg);
7803 %}
7804
7805 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7806 match(Set dst (CountLeadingZerosL src));
7807
7808 ins_cost(INSN_COST);
7809 format %{ "clz $dst, $src" %}
7810 ins_encode %{
7811 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7812 %}
7813
7814 ins_pipe(ialu_reg);
7815 %}
7816
7817 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7818 match(Set dst (CountTrailingZerosI src));
7819
7820 ins_cost(INSN_COST * 2);
7821 format %{ "rbitw $dst, $src\n\t"
7822 "clzw $dst, $dst" %}
7823 ins_encode %{
7824 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7825 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7826 %}
7827
7828 ins_pipe(ialu_reg);
7829 %}
7830
7831 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7832 match(Set dst (CountTrailingZerosL src));
7833
7834 ins_cost(INSN_COST * 2);
7835 format %{ "rbit $dst, $src\n\t"
7836 "clz $dst, $dst" %}
7837 ins_encode %{
7838 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7839 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7840 %}
7841
7842 ins_pipe(ialu_reg);
7843 %}
7844
7845 //---------- Population Count Instructions -------------------------------------
7846 //
7847
7848 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7849 match(Set dst (PopCountI src));
7850 effect(TEMP tmp);
7851 ins_cost(INSN_COST * 13);
7852
7853 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7854 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7855 "addv $tmp, $tmp\t# vector (8B)\n\t"
7856 "mov $dst, $tmp\t# vector (1D)" %}
7857 ins_encode %{
7858 __ fmovs($tmp$$FloatRegister, $src$$Register);
7859 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7860 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7861 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7862 %}
7863
7864 ins_pipe(pipe_class_default);
7865 %}
7866
7867 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7868 match(Set dst (PopCountI (LoadI mem)));
7869 effect(TEMP tmp);
7870 ins_cost(INSN_COST * 13);
7871
7872 format %{ "ldrs $tmp, $mem\n\t"
7873 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7874 "addv $tmp, $tmp\t# vector (8B)\n\t"
7875 "mov $dst, $tmp\t# vector (1D)" %}
7876 ins_encode %{
7877 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7878 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7879 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7880 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7881 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7882 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7883 %}
7884
7885 ins_pipe(pipe_class_default);
7886 %}
7887
7888 // Note: Long.bitCount(long) returns an int.
7889 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7890 match(Set dst (PopCountL src));
7891 effect(TEMP tmp);
7892 ins_cost(INSN_COST * 13);
7893
7894 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7895 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7896 "addv $tmp, $tmp\t# vector (8B)\n\t"
7897 "mov $dst, $tmp\t# vector (1D)" %}
7898 ins_encode %{
7899 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7900 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7901 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7902 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7903 %}
7904
7905 ins_pipe(pipe_class_default);
7906 %}
7907
7908 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7909 match(Set dst (PopCountL (LoadL mem)));
7910 effect(TEMP tmp);
7911 ins_cost(INSN_COST * 13);
7912
7913 format %{ "ldrd $tmp, $mem\n\t"
7914 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7915 "addv $tmp, $tmp\t# vector (8B)\n\t"
7916 "mov $dst, $tmp\t# vector (1D)" %}
7917 ins_encode %{
7918 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7919 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7921 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7922 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7923 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7924 %}
7925
7926 ins_pipe(pipe_class_default);
7927 %}
7928
7929 // ============================================================================
7930 // VerifyVectorAlignment Instruction
7931
7932 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7933 match(Set addr (VerifyVectorAlignment addr mask));
7934 effect(KILL cr);
7935 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7936 ins_encode %{
7937 Label Lskip;
7938 // check if masked bits of addr are zero
7939 __ tst($addr$$Register, $mask$$constant);
7940 __ br(Assembler::EQ, Lskip);
7941 __ stop("verify_vector_alignment found a misaligned vector memory access");
7942 __ bind(Lskip);
7943 %}
7944 ins_pipe(pipe_slow);
7945 %}
7946
7947 // ============================================================================
7948 // MemBar Instruction
7949
7950 instruct load_fence() %{
7951 match(LoadFence);
7952 ins_cost(VOLATILE_REF_COST);
7953
7954 format %{ "load_fence" %}
7955
7956 ins_encode %{
7957 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7958 %}
7959 ins_pipe(pipe_serial);
7960 %}
7961
7962 instruct unnecessary_membar_acquire() %{
7963 predicate(unnecessary_acquire(n));
7964 match(MemBarAcquire);
7965 ins_cost(0);
7966
7967 format %{ "membar_acquire (elided)" %}
7968
7969 ins_encode %{
7970 __ block_comment("membar_acquire (elided)");
7971 %}
7972
7973 ins_pipe(pipe_class_empty);
7974 %}
7975
7976 instruct membar_acquire() %{
7977 match(MemBarAcquire);
7978 ins_cost(VOLATILE_REF_COST);
7979
7980 format %{ "membar_acquire\n\t"
7981 "dmb ishld" %}
7982
7983 ins_encode %{
7984 __ block_comment("membar_acquire");
7985 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7986 %}
7987
7988 ins_pipe(pipe_serial);
7989 %}
7990
7991
7992 instruct membar_acquire_lock() %{
7993 match(MemBarAcquireLock);
7994 ins_cost(VOLATILE_REF_COST);
7995
7996 format %{ "membar_acquire_lock (elided)" %}
7997
7998 ins_encode %{
7999 __ block_comment("membar_acquire_lock (elided)");
8000 %}
8001
8002 ins_pipe(pipe_serial);
8003 %}
8004
8005 instruct store_fence() %{
8006 match(StoreFence);
8007 ins_cost(VOLATILE_REF_COST);
8008
8009 format %{ "store_fence" %}
8010
8011 ins_encode %{
8012 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8013 %}
8014 ins_pipe(pipe_serial);
8015 %}
8016
8017 instruct unnecessary_membar_release() %{
8018 predicate(unnecessary_release(n));
8019 match(MemBarRelease);
8020 ins_cost(0);
8021
8022 format %{ "membar_release (elided)" %}
8023
8024 ins_encode %{
8025 __ block_comment("membar_release (elided)");
8026 %}
8027 ins_pipe(pipe_serial);
8028 %}
8029
8030 instruct membar_release() %{
8031 match(MemBarRelease);
8032 ins_cost(VOLATILE_REF_COST);
8033
8034 format %{ "membar_release\n\t"
8035 "dmb ishst\n\tdmb ishld" %}
8036
8037 ins_encode %{
8038 __ block_comment("membar_release");
8039 // These will be merged if AlwaysMergeDMB is enabled.
8040 __ membar(Assembler::StoreStore);
8041 __ membar(Assembler::LoadStore);
8042 %}
8043 ins_pipe(pipe_serial);
8044 %}
8045
8046 instruct membar_storestore() %{
8047 match(MemBarStoreStore);
8048 match(StoreStoreFence);
8049 ins_cost(VOLATILE_REF_COST);
8050
8051 format %{ "MEMBAR-store-store" %}
8052
8053 ins_encode %{
8054 __ membar(Assembler::StoreStore);
8055 %}
8056 ins_pipe(pipe_serial);
8057 %}
8058
8059 instruct membar_release_lock() %{
8060 match(MemBarReleaseLock);
8061 ins_cost(VOLATILE_REF_COST);
8062
8063 format %{ "membar_release_lock (elided)" %}
8064
8065 ins_encode %{
8066 __ block_comment("membar_release_lock (elided)");
8067 %}
8068
8069 ins_pipe(pipe_serial);
8070 %}
8071
8072 instruct membar_storeload() %{
8073 match(MemBarStoreLoad);
8074 ins_cost(VOLATILE_REF_COST*100);
8075
8076 format %{ "MEMBAR-store-load\n\t"
8077 "dmb ish" %}
8078
8079 ins_encode %{
8080 __ block_comment("membar_storeload");
8081 __ membar(Assembler::StoreLoad);
8082 %}
8083
8084 ins_pipe(pipe_serial);
8085 %}
8086
8087 instruct unnecessary_membar_volatile() %{
8088 predicate(unnecessary_volatile(n));
8089 match(MemBarVolatile);
8090 ins_cost(0);
8091
8092 format %{ "membar_volatile (elided)" %}
8093
8094 ins_encode %{
8095 __ block_comment("membar_volatile (elided)");
8096 %}
8097
8098 ins_pipe(pipe_serial);
8099 %}
8100
8101 instruct membar_volatile() %{
8102 match(MemBarVolatile);
8103 ins_cost(VOLATILE_REF_COST*100);
8104
8105 format %{ "membar_volatile\n\t"
8106 "dmb ish"%}
8107
8108 ins_encode %{
8109 __ block_comment("membar_volatile");
8110 __ membar(Assembler::StoreLoad);
8111 %}
8112
8113 ins_pipe(pipe_serial);
8114 %}
8115
8116 instruct membar_full() %{
8117 match(MemBarFull);
8118 ins_cost(VOLATILE_REF_COST*100);
8119
8120 format %{ "membar_full\n\t"
8121 "dmb ish" %}
8122 ins_encode %{
8123 __ block_comment("membar_full");
8124 __ membar(Assembler::AnyAny);
8125 %}
8126
8127 ins_pipe(pipe_serial);
8128 %}
8129
8130 // ============================================================================
8131 // Cast/Convert Instructions
8132
8133 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8134 match(Set dst (CastX2P src));
8135
8136 ins_cost(INSN_COST);
8137 format %{ "mov $dst, $src\t# long -> ptr" %}
8138
8139 ins_encode %{
8140 if ($dst$$reg != $src$$reg) {
8141 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8142 }
8143 %}
8144
8145 ins_pipe(ialu_reg);
8146 %}
8147
8148 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8149 match(Set dst (CastI2N src));
8150
8151 ins_cost(INSN_COST);
8152 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8153
8154 ins_encode %{
8155 if ($dst$$reg != $src$$reg) {
8156 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8157 }
8158 %}
8159
8160 ins_pipe(ialu_reg);
8161 %}
8162
8163 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8164 match(Set dst (CastP2X src));
8165
8166 ins_cost(INSN_COST);
8167 format %{ "mov $dst, $src\t# ptr -> long" %}
8168
8169 ins_encode %{
8170 if ($dst$$reg != $src$$reg) {
8171 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8172 }
8173 %}
8174
8175 ins_pipe(ialu_reg);
8176 %}
8177
8178 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8179 match(Set dst (CastP2X src));
8180
8181 ins_cost(INSN_COST);
8182 format %{ "mov $dst, $src\t# ptr -> long" %}
8183
8184 ins_encode %{
8185 if ($dst$$reg != $src$$reg) {
8186 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8187 }
8188 %}
8189
8190 ins_pipe(ialu_reg);
8191 %}
8192
8193 // Convert oop into int for vectors alignment masking
8194 instruct convP2I(iRegINoSp dst, iRegP src) %{
8195 match(Set dst (ConvL2I (CastP2X src)));
8196
8197 ins_cost(INSN_COST);
8198 format %{ "movw $dst, $src\t# ptr -> int" %}
8199 ins_encode %{
8200 __ movw($dst$$Register, $src$$Register);
8201 %}
8202
8203 ins_pipe(ialu_reg);
8204 %}
8205
8206 // Convert compressed oop into int for vectors alignment masking
8207 // in case of 32bit oops (heap < 4Gb).
8208 instruct convN2I(iRegINoSp dst, iRegN src)
8209 %{
8210 predicate(CompressedOops::shift() == 0);
8211 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8212
8213 ins_cost(INSN_COST);
8214 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8215 ins_encode %{
8216 __ movw($dst$$Register, $src$$Register);
8217 %}
8218
8219 ins_pipe(ialu_reg);
8220 %}
8221
8222
8223 // Convert oop pointer into compressed form
8224 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8225 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8226 match(Set dst (EncodeP src));
8227 effect(KILL cr);
8228 ins_cost(INSN_COST * 3);
8229 format %{ "encode_heap_oop $dst, $src" %}
8230 ins_encode %{
8231 Register s = $src$$Register;
8232 Register d = $dst$$Register;
8233 __ encode_heap_oop(d, s);
8234 %}
8235 ins_pipe(ialu_reg);
8236 %}
8237
8238 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8239 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8240 match(Set dst (EncodeP src));
8241 ins_cost(INSN_COST * 3);
8242 format %{ "encode_heap_oop_not_null $dst, $src" %}
8243 ins_encode %{
8244 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8245 %}
8246 ins_pipe(ialu_reg);
8247 %}
8248
8249 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8250 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8251 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8252 match(Set dst (DecodeN src));
8253 ins_cost(INSN_COST * 3);
8254 format %{ "decode_heap_oop $dst, $src" %}
8255 ins_encode %{
8256 Register s = $src$$Register;
8257 Register d = $dst$$Register;
8258 __ decode_heap_oop(d, s);
8259 %}
8260 ins_pipe(ialu_reg);
8261 %}
8262
8263 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8264 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8265 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8266 match(Set dst (DecodeN src));
8267 ins_cost(INSN_COST * 3);
8268 format %{ "decode_heap_oop_not_null $dst, $src" %}
8269 ins_encode %{
8270 Register s = $src$$Register;
8271 Register d = $dst$$Register;
8272 __ decode_heap_oop_not_null(d, s);
8273 %}
8274 ins_pipe(ialu_reg);
8275 %}
8276
8277 // n.b. AArch64 implementations of encode_klass_not_null and
8278 // decode_klass_not_null do not modify the flags register so, unlike
8279 // Intel, we don't kill CR as a side effect here
8280
8281 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8282 match(Set dst (EncodePKlass src));
8283
8284 ins_cost(INSN_COST * 3);
8285 format %{ "encode_klass_not_null $dst,$src" %}
8286
8287 ins_encode %{
8288 Register src_reg = as_Register($src$$reg);
8289 Register dst_reg = as_Register($dst$$reg);
8290 __ encode_klass_not_null(dst_reg, src_reg);
8291 %}
8292
8293 ins_pipe(ialu_reg);
8294 %}
8295
8296 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8297 match(Set dst (DecodeNKlass src));
8298
8299 ins_cost(INSN_COST * 3);
8300 format %{ "decode_klass_not_null $dst,$src" %}
8301
8302 ins_encode %{
8303 Register src_reg = as_Register($src$$reg);
8304 Register dst_reg = as_Register($dst$$reg);
8305 if (dst_reg != src_reg) {
8306 __ decode_klass_not_null(dst_reg, src_reg);
8307 } else {
8308 __ decode_klass_not_null(dst_reg);
8309 }
8310 %}
8311
8312 ins_pipe(ialu_reg);
8313 %}
8314
8315 instruct checkCastPP(iRegPNoSp dst)
8316 %{
8317 match(Set dst (CheckCastPP dst));
8318
8319 size(0);
8320 format %{ "# checkcastPP of $dst" %}
8321 ins_encode(/* empty encoding */);
8322 ins_pipe(pipe_class_empty);
8323 %}
8324
8325 instruct castPP(iRegPNoSp dst)
8326 %{
8327 match(Set dst (CastPP dst));
8328
8329 size(0);
8330 format %{ "# castPP of $dst" %}
8331 ins_encode(/* empty encoding */);
8332 ins_pipe(pipe_class_empty);
8333 %}
8334
8335 instruct castII(iRegI dst)
8336 %{
8337 predicate(VerifyConstraintCasts == 0);
8338 match(Set dst (CastII dst));
8339
8340 size(0);
8341 format %{ "# castII of $dst" %}
8342 ins_encode(/* empty encoding */);
8343 ins_cost(0);
8344 ins_pipe(pipe_class_empty);
8345 %}
8346
8347 instruct castII_checked(iRegI dst, rFlagsReg cr)
8348 %{
8349 predicate(VerifyConstraintCasts > 0);
8350 match(Set dst (CastII dst));
8351 effect(KILL cr);
8352
8353 format %{ "# castII_checked of $dst" %}
8354 ins_encode %{
8355 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8356 %}
8357 ins_pipe(pipe_slow);
8358 %}
8359
8360 // The unchecked and checked variants for CastII below both use iRegINoSp for src and dst
8361 // as some consumers of CastII node like ConvHF2F forbid the stack pointer as an input
8362 // (please see convHF2F_reg_reg rule which requires input to be in an iRegINoSp register).
8363 instruct castII_nosp(iRegINoSp dst)
8364 %{
8365 predicate(VerifyConstraintCasts == 0);
8366 match(Set dst (CastII dst));
8367
8368 size(0);
8369 format %{ "# castII of $dst" %}
8370 ins_encode(/* empty encoding */);
8371 ins_cost(0);
8372 ins_pipe(pipe_class_empty);
8373 %}
8374
8375 instruct castII_checked_nosp(iRegINoSp dst, rFlagsReg cr)
8376 %{
8377 predicate(VerifyConstraintCasts > 0);
8378 match(Set dst (CastII dst));
8379 effect(KILL cr);
8380
8381 format %{ "# castII_checked of $dst" %}
8382 ins_encode %{
8383 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8384 %}
8385 ins_pipe(pipe_slow);
8386 %}
8387
8388 instruct castLL(iRegL dst)
8389 %{
8390 predicate(VerifyConstraintCasts == 0);
8391 match(Set dst (CastLL dst));
8392
8393 size(0);
8394 format %{ "# castLL of $dst" %}
8395 ins_encode(/* empty encoding */);
8396 ins_cost(0);
8397 ins_pipe(pipe_class_empty);
8398 %}
8399
8400 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8401 %{
8402 predicate(VerifyConstraintCasts > 0);
8403 match(Set dst (CastLL dst));
8404 effect(KILL cr);
8405
8406 format %{ "# castLL_checked of $dst" %}
8407 ins_encode %{
8408 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8409 %}
8410 ins_pipe(pipe_slow);
8411 %}
8412
8413 instruct castHH(vRegF dst)
8414 %{
8415 match(Set dst (CastHH dst));
8416 size(0);
8417 format %{ "# castHH of $dst" %}
8418 ins_encode(/* empty encoding */);
8419 ins_cost(0);
8420 ins_pipe(pipe_class_empty);
8421 %}
8422
8423 instruct castFF(vRegF dst)
8424 %{
8425 match(Set dst (CastFF dst));
8426
8427 size(0);
8428 format %{ "# castFF of $dst" %}
8429 ins_encode(/* empty encoding */);
8430 ins_cost(0);
8431 ins_pipe(pipe_class_empty);
8432 %}
8433
8434 instruct castDD(vRegD dst)
8435 %{
8436 match(Set dst (CastDD dst));
8437
8438 size(0);
8439 format %{ "# castDD of $dst" %}
8440 ins_encode(/* empty encoding */);
8441 ins_cost(0);
8442 ins_pipe(pipe_class_empty);
8443 %}
8444
8445 instruct castVV(vReg dst)
8446 %{
8447 match(Set dst (CastVV dst));
8448
8449 size(0);
8450 format %{ "# castVV of $dst" %}
8451 ins_encode(/* empty encoding */);
8452 ins_cost(0);
8453 ins_pipe(pipe_class_empty);
8454 %}
8455
8456 instruct castVVMask(pRegGov dst)
8457 %{
8458 match(Set dst (CastVV dst));
8459
8460 size(0);
8461 format %{ "# castVV of $dst" %}
8462 ins_encode(/* empty encoding */);
8463 ins_cost(0);
8464 ins_pipe(pipe_class_empty);
8465 %}
8466
8467 // Manifest a CmpU result in an integer register.
8468 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8469 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8470 %{
8471 match(Set dst (CmpU3 src1 src2));
8472 effect(KILL flags);
8473
8474 ins_cost(INSN_COST * 3);
8475 format %{
8476 "cmpw $src1, $src2\n\t"
8477 "csetw $dst, ne\n\t"
8478 "cnegw $dst, lo\t# CmpU3(reg)"
8479 %}
8480 ins_encode %{
8481 __ cmpw($src1$$Register, $src2$$Register);
8482 __ csetw($dst$$Register, Assembler::NE);
8483 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8484 %}
8485
8486 ins_pipe(pipe_class_default);
8487 %}
8488
8489 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8490 %{
8491 match(Set dst (CmpU3 src1 src2));
8492 effect(KILL flags);
8493
8494 ins_cost(INSN_COST * 3);
8495 format %{
8496 "subsw zr, $src1, $src2\n\t"
8497 "csetw $dst, ne\n\t"
8498 "cnegw $dst, lo\t# CmpU3(imm)"
8499 %}
8500 ins_encode %{
8501 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8502 __ csetw($dst$$Register, Assembler::NE);
8503 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8504 %}
8505
8506 ins_pipe(pipe_class_default);
8507 %}
8508
8509 // Manifest a CmpUL result in an integer register.
8510 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8511 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8512 %{
8513 match(Set dst (CmpUL3 src1 src2));
8514 effect(KILL flags);
8515
8516 ins_cost(INSN_COST * 3);
8517 format %{
8518 "cmp $src1, $src2\n\t"
8519 "csetw $dst, ne\n\t"
8520 "cnegw $dst, lo\t# CmpUL3(reg)"
8521 %}
8522 ins_encode %{
8523 __ cmp($src1$$Register, $src2$$Register);
8524 __ csetw($dst$$Register, Assembler::NE);
8525 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8526 %}
8527
8528 ins_pipe(pipe_class_default);
8529 %}
8530
8531 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8532 %{
8533 match(Set dst (CmpUL3 src1 src2));
8534 effect(KILL flags);
8535
8536 ins_cost(INSN_COST * 3);
8537 format %{
8538 "subs zr, $src1, $src2\n\t"
8539 "csetw $dst, ne\n\t"
8540 "cnegw $dst, lo\t# CmpUL3(imm)"
8541 %}
8542 ins_encode %{
8543 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8544 __ csetw($dst$$Register, Assembler::NE);
8545 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8546 %}
8547
8548 ins_pipe(pipe_class_default);
8549 %}
8550
8551 // Manifest a CmpL result in an integer register.
8552 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8553 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8554 %{
8555 match(Set dst (CmpL3 src1 src2));
8556 effect(KILL flags);
8557
8558 ins_cost(INSN_COST * 3);
8559 format %{
8560 "cmp $src1, $src2\n\t"
8561 "csetw $dst, ne\n\t"
8562 "cnegw $dst, lt\t# CmpL3(reg)"
8563 %}
8564 ins_encode %{
8565 __ cmp($src1$$Register, $src2$$Register);
8566 __ csetw($dst$$Register, Assembler::NE);
8567 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8568 %}
8569
8570 ins_pipe(pipe_class_default);
8571 %}
8572
8573 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8574 %{
8575 match(Set dst (CmpL3 src1 src2));
8576 effect(KILL flags);
8577
8578 ins_cost(INSN_COST * 3);
8579 format %{
8580 "subs zr, $src1, $src2\n\t"
8581 "csetw $dst, ne\n\t"
8582 "cnegw $dst, lt\t# CmpL3(imm)"
8583 %}
8584 ins_encode %{
8585 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8586 __ csetw($dst$$Register, Assembler::NE);
8587 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8588 %}
8589
8590 ins_pipe(pipe_class_default);
8591 %}
8592
8593 // ============================================================================
8594 // Conditional Move Instructions
8595
8596 // n.b. we have identical rules for both a signed compare op (cmpOp)
8597 // and an unsigned compare op (cmpOpU). it would be nice if we could
8598 // define an op class which merged both inputs and use it to type the
8599 // argument to a single rule. unfortunatelyt his fails because the
8600 // opclass does not live up to the COND_INTER interface of its
8601 // component operands. When the generic code tries to negate the
8602 // operand it ends up running the generci Machoper::negate method
8603 // which throws a ShouldNotHappen. So, we have to provide two flavours
8604 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8605
8606 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8607 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8608
8609 ins_cost(INSN_COST * 2);
8610 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8611
8612 ins_encode %{
8613 __ cselw(as_Register($dst$$reg),
8614 as_Register($src2$$reg),
8615 as_Register($src1$$reg),
8616 (Assembler::Condition)$cmp$$cmpcode);
8617 %}
8618
8619 ins_pipe(icond_reg_reg);
8620 %}
8621
8622 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8623 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8624
8625 ins_cost(INSN_COST * 2);
8626 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8627
8628 ins_encode %{
8629 __ cselw(as_Register($dst$$reg),
8630 as_Register($src2$$reg),
8631 as_Register($src1$$reg),
8632 (Assembler::Condition)$cmp$$cmpcode);
8633 %}
8634
8635 ins_pipe(icond_reg_reg);
8636 %}
8637
8638 // special cases where one arg is zero
8639
8640 // n.b. this is selected in preference to the rule above because it
8641 // avoids loading constant 0 into a source register
8642
8643 // TODO
8644 // we ought only to be able to cull one of these variants as the ideal
8645 // transforms ought always to order the zero consistently (to left/right?)
8646
8647 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8648 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8649
8650 ins_cost(INSN_COST * 2);
8651 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8652
8653 ins_encode %{
8654 __ cselw(as_Register($dst$$reg),
8655 as_Register($src$$reg),
8656 zr,
8657 (Assembler::Condition)$cmp$$cmpcode);
8658 %}
8659
8660 ins_pipe(icond_reg);
8661 %}
8662
8663 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8664 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8665
8666 ins_cost(INSN_COST * 2);
8667 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8668
8669 ins_encode %{
8670 __ cselw(as_Register($dst$$reg),
8671 as_Register($src$$reg),
8672 zr,
8673 (Assembler::Condition)$cmp$$cmpcode);
8674 %}
8675
8676 ins_pipe(icond_reg);
8677 %}
8678
8679 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8680 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8681
8682 ins_cost(INSN_COST * 2);
8683 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8684
8685 ins_encode %{
8686 __ cselw(as_Register($dst$$reg),
8687 zr,
8688 as_Register($src$$reg),
8689 (Assembler::Condition)$cmp$$cmpcode);
8690 %}
8691
8692 ins_pipe(icond_reg);
8693 %}
8694
8695 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8696 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8697
8698 ins_cost(INSN_COST * 2);
8699 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8700
8701 ins_encode %{
8702 __ cselw(as_Register($dst$$reg),
8703 zr,
8704 as_Register($src$$reg),
8705 (Assembler::Condition)$cmp$$cmpcode);
8706 %}
8707
8708 ins_pipe(icond_reg);
8709 %}
8710
8711 // special case for creating a boolean 0 or 1
8712
8713 // n.b. this is selected in preference to the rule above because it
8714 // avoids loading constants 0 and 1 into a source register
8715
8716 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8717 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8718
8719 ins_cost(INSN_COST * 2);
8720 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8721
8722 ins_encode %{
8723 // equivalently
8724 // cset(as_Register($dst$$reg),
8725 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8726 __ csincw(as_Register($dst$$reg),
8727 zr,
8728 zr,
8729 (Assembler::Condition)$cmp$$cmpcode);
8730 %}
8731
8732 ins_pipe(icond_none);
8733 %}
8734
8735 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8736 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8737
8738 ins_cost(INSN_COST * 2);
8739 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8740
8741 ins_encode %{
8742 // equivalently
8743 // cset(as_Register($dst$$reg),
8744 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8745 __ csincw(as_Register($dst$$reg),
8746 zr,
8747 zr,
8748 (Assembler::Condition)$cmp$$cmpcode);
8749 %}
8750
8751 ins_pipe(icond_none);
8752 %}
8753
8754 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8755 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8756
8757 ins_cost(INSN_COST * 2);
8758 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8759
8760 ins_encode %{
8761 __ csel(as_Register($dst$$reg),
8762 as_Register($src2$$reg),
8763 as_Register($src1$$reg),
8764 (Assembler::Condition)$cmp$$cmpcode);
8765 %}
8766
8767 ins_pipe(icond_reg_reg);
8768 %}
8769
8770 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8771 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8772
8773 ins_cost(INSN_COST * 2);
8774 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8775
8776 ins_encode %{
8777 __ csel(as_Register($dst$$reg),
8778 as_Register($src2$$reg),
8779 as_Register($src1$$reg),
8780 (Assembler::Condition)$cmp$$cmpcode);
8781 %}
8782
8783 ins_pipe(icond_reg_reg);
8784 %}
8785
8786 // special cases where one arg is zero
8787
8788 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8789 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8790
8791 ins_cost(INSN_COST * 2);
8792 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8793
8794 ins_encode %{
8795 __ csel(as_Register($dst$$reg),
8796 zr,
8797 as_Register($src$$reg),
8798 (Assembler::Condition)$cmp$$cmpcode);
8799 %}
8800
8801 ins_pipe(icond_reg);
8802 %}
8803
8804 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8805 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8806
8807 ins_cost(INSN_COST * 2);
8808 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8809
8810 ins_encode %{
8811 __ csel(as_Register($dst$$reg),
8812 zr,
8813 as_Register($src$$reg),
8814 (Assembler::Condition)$cmp$$cmpcode);
8815 %}
8816
8817 ins_pipe(icond_reg);
8818 %}
8819
8820 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8821 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8822
8823 ins_cost(INSN_COST * 2);
8824 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8825
8826 ins_encode %{
8827 __ csel(as_Register($dst$$reg),
8828 as_Register($src$$reg),
8829 zr,
8830 (Assembler::Condition)$cmp$$cmpcode);
8831 %}
8832
8833 ins_pipe(icond_reg);
8834 %}
8835
8836 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8837 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8838
8839 ins_cost(INSN_COST * 2);
8840 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8841
8842 ins_encode %{
8843 __ csel(as_Register($dst$$reg),
8844 as_Register($src$$reg),
8845 zr,
8846 (Assembler::Condition)$cmp$$cmpcode);
8847 %}
8848
8849 ins_pipe(icond_reg);
8850 %}
8851
8852 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8853 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8854
8855 ins_cost(INSN_COST * 2);
8856 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8857
8858 ins_encode %{
8859 __ csel(as_Register($dst$$reg),
8860 as_Register($src2$$reg),
8861 as_Register($src1$$reg),
8862 (Assembler::Condition)$cmp$$cmpcode);
8863 %}
8864
8865 ins_pipe(icond_reg_reg);
8866 %}
8867
8868 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8869 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8870
8871 ins_cost(INSN_COST * 2);
8872 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8873
8874 ins_encode %{
8875 __ csel(as_Register($dst$$reg),
8876 as_Register($src2$$reg),
8877 as_Register($src1$$reg),
8878 (Assembler::Condition)$cmp$$cmpcode);
8879 %}
8880
8881 ins_pipe(icond_reg_reg);
8882 %}
8883
8884 // special cases where one arg is zero
8885
8886 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8887 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8888
8889 ins_cost(INSN_COST * 2);
8890 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8891
8892 ins_encode %{
8893 __ csel(as_Register($dst$$reg),
8894 zr,
8895 as_Register($src$$reg),
8896 (Assembler::Condition)$cmp$$cmpcode);
8897 %}
8898
8899 ins_pipe(icond_reg);
8900 %}
8901
8902 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8903 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8904
8905 ins_cost(INSN_COST * 2);
8906 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8907
8908 ins_encode %{
8909 __ csel(as_Register($dst$$reg),
8910 zr,
8911 as_Register($src$$reg),
8912 (Assembler::Condition)$cmp$$cmpcode);
8913 %}
8914
8915 ins_pipe(icond_reg);
8916 %}
8917
8918 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8919 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8920
8921 ins_cost(INSN_COST * 2);
8922 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8923
8924 ins_encode %{
8925 __ csel(as_Register($dst$$reg),
8926 as_Register($src$$reg),
8927 zr,
8928 (Assembler::Condition)$cmp$$cmpcode);
8929 %}
8930
8931 ins_pipe(icond_reg);
8932 %}
8933
8934 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8935 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8936
8937 ins_cost(INSN_COST * 2);
8938 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8939
8940 ins_encode %{
8941 __ csel(as_Register($dst$$reg),
8942 as_Register($src$$reg),
8943 zr,
8944 (Assembler::Condition)$cmp$$cmpcode);
8945 %}
8946
8947 ins_pipe(icond_reg);
8948 %}
8949
8950 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8951 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8952
8953 ins_cost(INSN_COST * 2);
8954 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8955
8956 ins_encode %{
8957 __ cselw(as_Register($dst$$reg),
8958 as_Register($src2$$reg),
8959 as_Register($src1$$reg),
8960 (Assembler::Condition)$cmp$$cmpcode);
8961 %}
8962
8963 ins_pipe(icond_reg_reg);
8964 %}
8965
8966 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8967 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8968
8969 ins_cost(INSN_COST * 2);
8970 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8971
8972 ins_encode %{
8973 __ cselw(as_Register($dst$$reg),
8974 as_Register($src2$$reg),
8975 as_Register($src1$$reg),
8976 (Assembler::Condition)$cmp$$cmpcode);
8977 %}
8978
8979 ins_pipe(icond_reg_reg);
8980 %}
8981
8982 // special cases where one arg is zero
8983
8984 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8985 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8986
8987 ins_cost(INSN_COST * 2);
8988 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8989
8990 ins_encode %{
8991 __ cselw(as_Register($dst$$reg),
8992 zr,
8993 as_Register($src$$reg),
8994 (Assembler::Condition)$cmp$$cmpcode);
8995 %}
8996
8997 ins_pipe(icond_reg);
8998 %}
8999
9000 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9001 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9002
9003 ins_cost(INSN_COST * 2);
9004 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9005
9006 ins_encode %{
9007 __ cselw(as_Register($dst$$reg),
9008 zr,
9009 as_Register($src$$reg),
9010 (Assembler::Condition)$cmp$$cmpcode);
9011 %}
9012
9013 ins_pipe(icond_reg);
9014 %}
9015
9016 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9017 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9018
9019 ins_cost(INSN_COST * 2);
9020 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9021
9022 ins_encode %{
9023 __ cselw(as_Register($dst$$reg),
9024 as_Register($src$$reg),
9025 zr,
9026 (Assembler::Condition)$cmp$$cmpcode);
9027 %}
9028
9029 ins_pipe(icond_reg);
9030 %}
9031
9032 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9033 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9034
9035 ins_cost(INSN_COST * 2);
9036 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9037
9038 ins_encode %{
9039 __ cselw(as_Register($dst$$reg),
9040 as_Register($src$$reg),
9041 zr,
9042 (Assembler::Condition)$cmp$$cmpcode);
9043 %}
9044
9045 ins_pipe(icond_reg);
9046 %}
9047
9048 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9049 %{
9050 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9051
9052 ins_cost(INSN_COST * 3);
9053
9054 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9055 ins_encode %{
9056 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9057 __ fcsels(as_FloatRegister($dst$$reg),
9058 as_FloatRegister($src2$$reg),
9059 as_FloatRegister($src1$$reg),
9060 cond);
9061 %}
9062
9063 ins_pipe(fp_cond_reg_reg_s);
9064 %}
9065
9066 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9067 %{
9068 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9069
9070 ins_cost(INSN_COST * 3);
9071
9072 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9073 ins_encode %{
9074 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9075 __ fcsels(as_FloatRegister($dst$$reg),
9076 as_FloatRegister($src2$$reg),
9077 as_FloatRegister($src1$$reg),
9078 cond);
9079 %}
9080
9081 ins_pipe(fp_cond_reg_reg_s);
9082 %}
9083
9084 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9085 %{
9086 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9087
9088 ins_cost(INSN_COST * 3);
9089
9090 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9091 ins_encode %{
9092 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9093 __ fcseld(as_FloatRegister($dst$$reg),
9094 as_FloatRegister($src2$$reg),
9095 as_FloatRegister($src1$$reg),
9096 cond);
9097 %}
9098
9099 ins_pipe(fp_cond_reg_reg_d);
9100 %}
9101
9102 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9103 %{
9104 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9105
9106 ins_cost(INSN_COST * 3);
9107
9108 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9109 ins_encode %{
9110 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9111 __ fcseld(as_FloatRegister($dst$$reg),
9112 as_FloatRegister($src2$$reg),
9113 as_FloatRegister($src1$$reg),
9114 cond);
9115 %}
9116
9117 ins_pipe(fp_cond_reg_reg_d);
9118 %}
9119
9120 // ============================================================================
9121 // Arithmetic Instructions
9122 //
9123
9124 // Integer Addition
9125
9126 // TODO
9127 // these currently employ operations which do not set CR and hence are
9128 // not flagged as killing CR but we would like to isolate the cases
9129 // where we want to set flags from those where we don't. need to work
9130 // out how to do that.
9131
9132 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9133 match(Set dst (AddI src1 src2));
9134
9135 ins_cost(INSN_COST);
9136 format %{ "addw $dst, $src1, $src2" %}
9137
9138 ins_encode %{
9139 __ addw(as_Register($dst$$reg),
9140 as_Register($src1$$reg),
9141 as_Register($src2$$reg));
9142 %}
9143
9144 ins_pipe(ialu_reg_reg);
9145 %}
9146
9147 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9148 match(Set dst (AddI src1 src2));
9149
9150 ins_cost(INSN_COST);
9151 format %{ "addw $dst, $src1, $src2" %}
9152
9153 // use opcode to indicate that this is an add not a sub
9154 opcode(0x0);
9155
9156 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9157
9158 ins_pipe(ialu_reg_imm);
9159 %}
9160
9161 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9162 match(Set dst (AddI (ConvL2I src1) src2));
9163
9164 ins_cost(INSN_COST);
9165 format %{ "addw $dst, $src1, $src2" %}
9166
9167 // use opcode to indicate that this is an add not a sub
9168 opcode(0x0);
9169
9170 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9171
9172 ins_pipe(ialu_reg_imm);
9173 %}
9174
9175 // Pointer Addition
9176 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9177 match(Set dst (AddP src1 src2));
9178
9179 ins_cost(INSN_COST);
9180 format %{ "add $dst, $src1, $src2\t# ptr" %}
9181
9182 ins_encode %{
9183 __ add(as_Register($dst$$reg),
9184 as_Register($src1$$reg),
9185 as_Register($src2$$reg));
9186 %}
9187
9188 ins_pipe(ialu_reg_reg);
9189 %}
9190
9191 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9192 match(Set dst (AddP src1 (ConvI2L src2)));
9193
9194 ins_cost(1.9 * INSN_COST);
9195 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9196
9197 ins_encode %{
9198 __ add(as_Register($dst$$reg),
9199 as_Register($src1$$reg),
9200 as_Register($src2$$reg), ext::sxtw);
9201 %}
9202
9203 ins_pipe(ialu_reg_reg);
9204 %}
9205
9206 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9207 match(Set dst (AddP src1 (LShiftL src2 scale)));
9208
9209 ins_cost(1.9 * INSN_COST);
9210 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9211
9212 ins_encode %{
9213 __ lea(as_Register($dst$$reg),
9214 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9215 Address::lsl($scale$$constant)));
9216 %}
9217
9218 ins_pipe(ialu_reg_reg_shift);
9219 %}
9220
9221 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9222 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9223
9224 ins_cost(1.9 * INSN_COST);
9225 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9226
9227 ins_encode %{
9228 __ lea(as_Register($dst$$reg),
9229 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9230 Address::sxtw($scale$$constant)));
9231 %}
9232
9233 ins_pipe(ialu_reg_reg_shift);
9234 %}
9235
9236 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9237 match(Set dst (LShiftL (ConvI2L src) scale));
9238
9239 ins_cost(INSN_COST);
9240 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9241
9242 ins_encode %{
9243 __ sbfiz(as_Register($dst$$reg),
9244 as_Register($src$$reg),
9245 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9246 %}
9247
9248 ins_pipe(ialu_reg_shift);
9249 %}
9250
9251 // Pointer Immediate Addition
9252 // n.b. this needs to be more expensive than using an indirect memory
9253 // operand
9254 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9255 match(Set dst (AddP src1 src2));
9256
9257 ins_cost(INSN_COST);
9258 format %{ "add $dst, $src1, $src2\t# ptr" %}
9259
9260 // use opcode to indicate that this is an add not a sub
9261 opcode(0x0);
9262
9263 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9264
9265 ins_pipe(ialu_reg_imm);
9266 %}
9267
9268 // Long Addition
9269 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9270
9271 match(Set dst (AddL src1 src2));
9272
9273 ins_cost(INSN_COST);
9274 format %{ "add $dst, $src1, $src2" %}
9275
9276 ins_encode %{
9277 __ add(as_Register($dst$$reg),
9278 as_Register($src1$$reg),
9279 as_Register($src2$$reg));
9280 %}
9281
9282 ins_pipe(ialu_reg_reg);
9283 %}
9284
9285 // No constant pool entries requiredLong Immediate Addition.
9286 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9287 match(Set dst (AddL src1 src2));
9288
9289 ins_cost(INSN_COST);
9290 format %{ "add $dst, $src1, $src2" %}
9291
9292 // use opcode to indicate that this is an add not a sub
9293 opcode(0x0);
9294
9295 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9296
9297 ins_pipe(ialu_reg_imm);
9298 %}
9299
9300 // Integer Subtraction
9301 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9302 match(Set dst (SubI src1 src2));
9303
9304 ins_cost(INSN_COST);
9305 format %{ "subw $dst, $src1, $src2" %}
9306
9307 ins_encode %{
9308 __ subw(as_Register($dst$$reg),
9309 as_Register($src1$$reg),
9310 as_Register($src2$$reg));
9311 %}
9312
9313 ins_pipe(ialu_reg_reg);
9314 %}
9315
9316 // Immediate Subtraction
9317 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9318 match(Set dst (SubI src1 src2));
9319
9320 ins_cost(INSN_COST);
9321 format %{ "subw $dst, $src1, $src2" %}
9322
9323 // use opcode to indicate that this is a sub not an add
9324 opcode(0x1);
9325
9326 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9327
9328 ins_pipe(ialu_reg_imm);
9329 %}
9330
9331 // Long Subtraction
9332 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9333
9334 match(Set dst (SubL src1 src2));
9335
9336 ins_cost(INSN_COST);
9337 format %{ "sub $dst, $src1, $src2" %}
9338
9339 ins_encode %{
9340 __ sub(as_Register($dst$$reg),
9341 as_Register($src1$$reg),
9342 as_Register($src2$$reg));
9343 %}
9344
9345 ins_pipe(ialu_reg_reg);
9346 %}
9347
9348 // No constant pool entries requiredLong Immediate Subtraction.
9349 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9350 match(Set dst (SubL src1 src2));
9351
9352 ins_cost(INSN_COST);
9353 format %{ "sub$dst, $src1, $src2" %}
9354
9355 // use opcode to indicate that this is a sub not an add
9356 opcode(0x1);
9357
9358 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9359
9360 ins_pipe(ialu_reg_imm);
9361 %}
9362
9363 // Integer Negation (special case for sub)
9364
9365 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9366 match(Set dst (SubI zero src));
9367
9368 ins_cost(INSN_COST);
9369 format %{ "negw $dst, $src\t# int" %}
9370
9371 ins_encode %{
9372 __ negw(as_Register($dst$$reg),
9373 as_Register($src$$reg));
9374 %}
9375
9376 ins_pipe(ialu_reg);
9377 %}
9378
9379 // Long Negation
9380
9381 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9382 match(Set dst (SubL zero src));
9383
9384 ins_cost(INSN_COST);
9385 format %{ "neg $dst, $src\t# long" %}
9386
9387 ins_encode %{
9388 __ neg(as_Register($dst$$reg),
9389 as_Register($src$$reg));
9390 %}
9391
9392 ins_pipe(ialu_reg);
9393 %}
9394
9395 // Integer Multiply
9396
9397 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9398 match(Set dst (MulI src1 src2));
9399
9400 ins_cost(INSN_COST * 3);
9401 format %{ "mulw $dst, $src1, $src2" %}
9402
9403 ins_encode %{
9404 __ mulw(as_Register($dst$$reg),
9405 as_Register($src1$$reg),
9406 as_Register($src2$$reg));
9407 %}
9408
9409 ins_pipe(imul_reg_reg);
9410 %}
9411
9412 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9413 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9414
9415 ins_cost(INSN_COST * 3);
9416 format %{ "smull $dst, $src1, $src2" %}
9417
9418 ins_encode %{
9419 __ smull(as_Register($dst$$reg),
9420 as_Register($src1$$reg),
9421 as_Register($src2$$reg));
9422 %}
9423
9424 ins_pipe(imul_reg_reg);
9425 %}
9426
9427 // Long Multiply
9428
9429 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9430 match(Set dst (MulL src1 src2));
9431
9432 ins_cost(INSN_COST * 5);
9433 format %{ "mul $dst, $src1, $src2" %}
9434
9435 ins_encode %{
9436 __ mul(as_Register($dst$$reg),
9437 as_Register($src1$$reg),
9438 as_Register($src2$$reg));
9439 %}
9440
9441 ins_pipe(lmul_reg_reg);
9442 %}
9443
9444 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9445 %{
9446 match(Set dst (MulHiL src1 src2));
9447
9448 ins_cost(INSN_COST * 7);
9449 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9450
9451 ins_encode %{
9452 __ smulh(as_Register($dst$$reg),
9453 as_Register($src1$$reg),
9454 as_Register($src2$$reg));
9455 %}
9456
9457 ins_pipe(lmul_reg_reg);
9458 %}
9459
9460 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9461 %{
9462 match(Set dst (UMulHiL src1 src2));
9463
9464 ins_cost(INSN_COST * 7);
9465 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9466
9467 ins_encode %{
9468 __ umulh(as_Register($dst$$reg),
9469 as_Register($src1$$reg),
9470 as_Register($src2$$reg));
9471 %}
9472
9473 ins_pipe(lmul_reg_reg);
9474 %}
9475
9476 // Combined Integer Multiply & Add/Sub
9477
9478 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9479 match(Set dst (AddI src3 (MulI src1 src2)));
9480
9481 ins_cost(INSN_COST * 3);
9482 format %{ "madd $dst, $src1, $src2, $src3" %}
9483
9484 ins_encode %{
9485 __ maddw(as_Register($dst$$reg),
9486 as_Register($src1$$reg),
9487 as_Register($src2$$reg),
9488 as_Register($src3$$reg));
9489 %}
9490
9491 ins_pipe(imac_reg_reg);
9492 %}
9493
9494 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9495 match(Set dst (SubI src3 (MulI src1 src2)));
9496
9497 ins_cost(INSN_COST * 3);
9498 format %{ "msub $dst, $src1, $src2, $src3" %}
9499
9500 ins_encode %{
9501 __ msubw(as_Register($dst$$reg),
9502 as_Register($src1$$reg),
9503 as_Register($src2$$reg),
9504 as_Register($src3$$reg));
9505 %}
9506
9507 ins_pipe(imac_reg_reg);
9508 %}
9509
9510 // Combined Integer Multiply & Neg
9511
9512 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9513 match(Set dst (MulI (SubI zero src1) src2));
9514
9515 ins_cost(INSN_COST * 3);
9516 format %{ "mneg $dst, $src1, $src2" %}
9517
9518 ins_encode %{
9519 __ mnegw(as_Register($dst$$reg),
9520 as_Register($src1$$reg),
9521 as_Register($src2$$reg));
9522 %}
9523
9524 ins_pipe(imac_reg_reg);
9525 %}
9526
9527 // Combined Long Multiply & Add/Sub
9528
9529 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9530 match(Set dst (AddL src3 (MulL src1 src2)));
9531
9532 ins_cost(INSN_COST * 5);
9533 format %{ "madd $dst, $src1, $src2, $src3" %}
9534
9535 ins_encode %{
9536 __ madd(as_Register($dst$$reg),
9537 as_Register($src1$$reg),
9538 as_Register($src2$$reg),
9539 as_Register($src3$$reg));
9540 %}
9541
9542 ins_pipe(lmac_reg_reg);
9543 %}
9544
9545 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9546 match(Set dst (SubL src3 (MulL src1 src2)));
9547
9548 ins_cost(INSN_COST * 5);
9549 format %{ "msub $dst, $src1, $src2, $src3" %}
9550
9551 ins_encode %{
9552 __ msub(as_Register($dst$$reg),
9553 as_Register($src1$$reg),
9554 as_Register($src2$$reg),
9555 as_Register($src3$$reg));
9556 %}
9557
9558 ins_pipe(lmac_reg_reg);
9559 %}
9560
9561 // Combined Long Multiply & Neg
9562
9563 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9564 match(Set dst (MulL (SubL zero src1) src2));
9565
9566 ins_cost(INSN_COST * 5);
9567 format %{ "mneg $dst, $src1, $src2" %}
9568
9569 ins_encode %{
9570 __ mneg(as_Register($dst$$reg),
9571 as_Register($src1$$reg),
9572 as_Register($src2$$reg));
9573 %}
9574
9575 ins_pipe(lmac_reg_reg);
9576 %}
9577
9578 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9579
9580 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9581 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9582
9583 ins_cost(INSN_COST * 3);
9584 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9585
9586 ins_encode %{
9587 __ smaddl(as_Register($dst$$reg),
9588 as_Register($src1$$reg),
9589 as_Register($src2$$reg),
9590 as_Register($src3$$reg));
9591 %}
9592
9593 ins_pipe(imac_reg_reg);
9594 %}
9595
9596 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9597 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9598
9599 ins_cost(INSN_COST * 3);
9600 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9601
9602 ins_encode %{
9603 __ smsubl(as_Register($dst$$reg),
9604 as_Register($src1$$reg),
9605 as_Register($src2$$reg),
9606 as_Register($src3$$reg));
9607 %}
9608
9609 ins_pipe(imac_reg_reg);
9610 %}
9611
9612 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9613 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9614
9615 ins_cost(INSN_COST * 3);
9616 format %{ "smnegl $dst, $src1, $src2" %}
9617
9618 ins_encode %{
9619 __ smnegl(as_Register($dst$$reg),
9620 as_Register($src1$$reg),
9621 as_Register($src2$$reg));
9622 %}
9623
9624 ins_pipe(imac_reg_reg);
9625 %}
9626
9627 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9628
9629 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9630 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9631
9632 ins_cost(INSN_COST * 5);
9633 format %{ "mulw rscratch1, $src1, $src2\n\t"
9634 "maddw $dst, $src3, $src4, rscratch1" %}
9635
9636 ins_encode %{
9637 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9638 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9639
9640 ins_pipe(imac_reg_reg);
9641 %}
9642
9643 // Integer Divide
9644
9645 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9646 match(Set dst (DivI src1 src2));
9647
9648 ins_cost(INSN_COST * 19);
9649 format %{ "sdivw $dst, $src1, $src2" %}
9650
9651 ins_encode(aarch64_enc_divw(dst, src1, src2));
9652 ins_pipe(idiv_reg_reg);
9653 %}
9654
9655 // Long Divide
9656
9657 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9658 match(Set dst (DivL src1 src2));
9659
9660 ins_cost(INSN_COST * 35);
9661 format %{ "sdiv $dst, $src1, $src2" %}
9662
9663 ins_encode(aarch64_enc_div(dst, src1, src2));
9664 ins_pipe(ldiv_reg_reg);
9665 %}
9666
9667 // Integer Remainder
9668
9669 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9670 match(Set dst (ModI src1 src2));
9671
9672 ins_cost(INSN_COST * 22);
9673 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9674 "msubw $dst, rscratch1, $src2, $src1" %}
9675
9676 ins_encode(aarch64_enc_modw(dst, src1, src2));
9677 ins_pipe(idiv_reg_reg);
9678 %}
9679
9680 // Long Remainder
9681
9682 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9683 match(Set dst (ModL src1 src2));
9684
9685 ins_cost(INSN_COST * 38);
9686 format %{ "sdiv rscratch1, $src1, $src2\n"
9687 "msub $dst, rscratch1, $src2, $src1" %}
9688
9689 ins_encode(aarch64_enc_mod(dst, src1, src2));
9690 ins_pipe(ldiv_reg_reg);
9691 %}
9692
9693 // Unsigned Integer Divide
9694
9695 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9696 match(Set dst (UDivI src1 src2));
9697
9698 ins_cost(INSN_COST * 19);
9699 format %{ "udivw $dst, $src1, $src2" %}
9700
9701 ins_encode %{
9702 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9703 %}
9704
9705 ins_pipe(idiv_reg_reg);
9706 %}
9707
9708 // Unsigned Long Divide
9709
9710 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9711 match(Set dst (UDivL src1 src2));
9712
9713 ins_cost(INSN_COST * 35);
9714 format %{ "udiv $dst, $src1, $src2" %}
9715
9716 ins_encode %{
9717 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9718 %}
9719
9720 ins_pipe(ldiv_reg_reg);
9721 %}
9722
9723 // Unsigned Integer Remainder
9724
9725 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9726 match(Set dst (UModI src1 src2));
9727
9728 ins_cost(INSN_COST * 22);
9729 format %{ "udivw rscratch1, $src1, $src2\n\t"
9730 "msubw $dst, rscratch1, $src2, $src1" %}
9731
9732 ins_encode %{
9733 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9734 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9735 %}
9736
9737 ins_pipe(idiv_reg_reg);
9738 %}
9739
9740 // Unsigned Long Remainder
9741
9742 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9743 match(Set dst (UModL src1 src2));
9744
9745 ins_cost(INSN_COST * 38);
9746 format %{ "udiv rscratch1, $src1, $src2\n"
9747 "msub $dst, rscratch1, $src2, $src1" %}
9748
9749 ins_encode %{
9750 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9751 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9752 %}
9753
9754 ins_pipe(ldiv_reg_reg);
9755 %}
9756
9757 // Integer Shifts
9758
9759 // Shift Left Register
9760 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9761 match(Set dst (LShiftI src1 src2));
9762
9763 ins_cost(INSN_COST * 2);
9764 format %{ "lslvw $dst, $src1, $src2" %}
9765
9766 ins_encode %{
9767 __ lslvw(as_Register($dst$$reg),
9768 as_Register($src1$$reg),
9769 as_Register($src2$$reg));
9770 %}
9771
9772 ins_pipe(ialu_reg_reg_vshift);
9773 %}
9774
9775 // Shift Left Immediate
9776 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9777 match(Set dst (LShiftI src1 src2));
9778
9779 ins_cost(INSN_COST);
9780 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9781
9782 ins_encode %{
9783 __ lslw(as_Register($dst$$reg),
9784 as_Register($src1$$reg),
9785 $src2$$constant & 0x1f);
9786 %}
9787
9788 ins_pipe(ialu_reg_shift);
9789 %}
9790
9791 // Shift Right Logical Register
9792 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9793 match(Set dst (URShiftI src1 src2));
9794
9795 ins_cost(INSN_COST * 2);
9796 format %{ "lsrvw $dst, $src1, $src2" %}
9797
9798 ins_encode %{
9799 __ lsrvw(as_Register($dst$$reg),
9800 as_Register($src1$$reg),
9801 as_Register($src2$$reg));
9802 %}
9803
9804 ins_pipe(ialu_reg_reg_vshift);
9805 %}
9806
9807 // Shift Right Logical Immediate
9808 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9809 match(Set dst (URShiftI src1 src2));
9810
9811 ins_cost(INSN_COST);
9812 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9813
9814 ins_encode %{
9815 __ lsrw(as_Register($dst$$reg),
9816 as_Register($src1$$reg),
9817 $src2$$constant & 0x1f);
9818 %}
9819
9820 ins_pipe(ialu_reg_shift);
9821 %}
9822
9823 // Shift Right Arithmetic Register
9824 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9825 match(Set dst (RShiftI src1 src2));
9826
9827 ins_cost(INSN_COST * 2);
9828 format %{ "asrvw $dst, $src1, $src2" %}
9829
9830 ins_encode %{
9831 __ asrvw(as_Register($dst$$reg),
9832 as_Register($src1$$reg),
9833 as_Register($src2$$reg));
9834 %}
9835
9836 ins_pipe(ialu_reg_reg_vshift);
9837 %}
9838
9839 // Shift Right Arithmetic Immediate
9840 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9841 match(Set dst (RShiftI src1 src2));
9842
9843 ins_cost(INSN_COST);
9844 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9845
9846 ins_encode %{
9847 __ asrw(as_Register($dst$$reg),
9848 as_Register($src1$$reg),
9849 $src2$$constant & 0x1f);
9850 %}
9851
9852 ins_pipe(ialu_reg_shift);
9853 %}
9854
9855 // Combined Int Mask and Right Shift (using UBFM)
9856 // TODO
9857
9858 // Long Shifts
9859
9860 // Shift Left Register
9861 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9862 match(Set dst (LShiftL src1 src2));
9863
9864 ins_cost(INSN_COST * 2);
9865 format %{ "lslv $dst, $src1, $src2" %}
9866
9867 ins_encode %{
9868 __ lslv(as_Register($dst$$reg),
9869 as_Register($src1$$reg),
9870 as_Register($src2$$reg));
9871 %}
9872
9873 ins_pipe(ialu_reg_reg_vshift);
9874 %}
9875
9876 // Shift Left Immediate
9877 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9878 match(Set dst (LShiftL src1 src2));
9879
9880 ins_cost(INSN_COST);
9881 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9882
9883 ins_encode %{
9884 __ lsl(as_Register($dst$$reg),
9885 as_Register($src1$$reg),
9886 $src2$$constant & 0x3f);
9887 %}
9888
9889 ins_pipe(ialu_reg_shift);
9890 %}
9891
9892 // Shift Right Logical Register
9893 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9894 match(Set dst (URShiftL src1 src2));
9895
9896 ins_cost(INSN_COST * 2);
9897 format %{ "lsrv $dst, $src1, $src2" %}
9898
9899 ins_encode %{
9900 __ lsrv(as_Register($dst$$reg),
9901 as_Register($src1$$reg),
9902 as_Register($src2$$reg));
9903 %}
9904
9905 ins_pipe(ialu_reg_reg_vshift);
9906 %}
9907
9908 // Shift Right Logical Immediate
9909 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9910 match(Set dst (URShiftL src1 src2));
9911
9912 ins_cost(INSN_COST);
9913 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9914
9915 ins_encode %{
9916 __ lsr(as_Register($dst$$reg),
9917 as_Register($src1$$reg),
9918 $src2$$constant & 0x3f);
9919 %}
9920
9921 ins_pipe(ialu_reg_shift);
9922 %}
9923
9924 // A special-case pattern for card table stores.
9925 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9926 match(Set dst (URShiftL (CastP2X src1) src2));
9927
9928 ins_cost(INSN_COST);
9929 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9930
9931 ins_encode %{
9932 __ lsr(as_Register($dst$$reg),
9933 as_Register($src1$$reg),
9934 $src2$$constant & 0x3f);
9935 %}
9936
9937 ins_pipe(ialu_reg_shift);
9938 %}
9939
9940 // Shift Right Arithmetic Register
9941 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9942 match(Set dst (RShiftL src1 src2));
9943
9944 ins_cost(INSN_COST * 2);
9945 format %{ "asrv $dst, $src1, $src2" %}
9946
9947 ins_encode %{
9948 __ asrv(as_Register($dst$$reg),
9949 as_Register($src1$$reg),
9950 as_Register($src2$$reg));
9951 %}
9952
9953 ins_pipe(ialu_reg_reg_vshift);
9954 %}
9955
9956 // Shift Right Arithmetic Immediate
9957 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9958 match(Set dst (RShiftL src1 src2));
9959
9960 ins_cost(INSN_COST);
9961 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9962
9963 ins_encode %{
9964 __ asr(as_Register($dst$$reg),
9965 as_Register($src1$$reg),
9966 $src2$$constant & 0x3f);
9967 %}
9968
9969 ins_pipe(ialu_reg_shift);
9970 %}
9971
9972 // BEGIN This section of the file is automatically generated. Do not edit --------------
9973 // This section is generated from aarch64_ad.m4
9974
9975 // This pattern is automatically generated from aarch64_ad.m4
9976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9977 instruct regL_not_reg(iRegLNoSp dst,
9978 iRegL src1, immL_M1 m1,
9979 rFlagsReg cr) %{
9980 match(Set dst (XorL src1 m1));
9981 ins_cost(INSN_COST);
9982 format %{ "eon $dst, $src1, zr" %}
9983
9984 ins_encode %{
9985 __ eon(as_Register($dst$$reg),
9986 as_Register($src1$$reg),
9987 zr,
9988 Assembler::LSL, 0);
9989 %}
9990
9991 ins_pipe(ialu_reg);
9992 %}
9993
9994 // This pattern is automatically generated from aarch64_ad.m4
9995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9996 instruct regI_not_reg(iRegINoSp dst,
9997 iRegIorL2I src1, immI_M1 m1,
9998 rFlagsReg cr) %{
9999 match(Set dst (XorI src1 m1));
10000 ins_cost(INSN_COST);
10001 format %{ "eonw $dst, $src1, zr" %}
10002
10003 ins_encode %{
10004 __ eonw(as_Register($dst$$reg),
10005 as_Register($src1$$reg),
10006 zr,
10007 Assembler::LSL, 0);
10008 %}
10009
10010 ins_pipe(ialu_reg);
10011 %}
10012
10013 // This pattern is automatically generated from aarch64_ad.m4
10014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10015 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10016 immI0 zero, iRegIorL2I src1, immI src2) %{
10017 match(Set dst (SubI zero (URShiftI src1 src2)));
10018
10019 ins_cost(1.9 * INSN_COST);
10020 format %{ "negw $dst, $src1, LSR $src2" %}
10021
10022 ins_encode %{
10023 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10024 Assembler::LSR, $src2$$constant & 0x1f);
10025 %}
10026
10027 ins_pipe(ialu_reg_shift);
10028 %}
10029
10030 // This pattern is automatically generated from aarch64_ad.m4
10031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10032 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10033 immI0 zero, iRegIorL2I src1, immI src2) %{
10034 match(Set dst (SubI zero (RShiftI src1 src2)));
10035
10036 ins_cost(1.9 * INSN_COST);
10037 format %{ "negw $dst, $src1, ASR $src2" %}
10038
10039 ins_encode %{
10040 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10041 Assembler::ASR, $src2$$constant & 0x1f);
10042 %}
10043
10044 ins_pipe(ialu_reg_shift);
10045 %}
10046
10047 // This pattern is automatically generated from aarch64_ad.m4
10048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10049 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10050 immI0 zero, iRegIorL2I src1, immI src2) %{
10051 match(Set dst (SubI zero (LShiftI src1 src2)));
10052
10053 ins_cost(1.9 * INSN_COST);
10054 format %{ "negw $dst, $src1, LSL $src2" %}
10055
10056 ins_encode %{
10057 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10058 Assembler::LSL, $src2$$constant & 0x1f);
10059 %}
10060
10061 ins_pipe(ialu_reg_shift);
10062 %}
10063
10064 // This pattern is automatically generated from aarch64_ad.m4
10065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10066 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10067 immL0 zero, iRegL src1, immI src2) %{
10068 match(Set dst (SubL zero (URShiftL src1 src2)));
10069
10070 ins_cost(1.9 * INSN_COST);
10071 format %{ "neg $dst, $src1, LSR $src2" %}
10072
10073 ins_encode %{
10074 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10075 Assembler::LSR, $src2$$constant & 0x3f);
10076 %}
10077
10078 ins_pipe(ialu_reg_shift);
10079 %}
10080
10081 // This pattern is automatically generated from aarch64_ad.m4
10082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10083 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10084 immL0 zero, iRegL src1, immI src2) %{
10085 match(Set dst (SubL zero (RShiftL src1 src2)));
10086
10087 ins_cost(1.9 * INSN_COST);
10088 format %{ "neg $dst, $src1, ASR $src2" %}
10089
10090 ins_encode %{
10091 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10092 Assembler::ASR, $src2$$constant & 0x3f);
10093 %}
10094
10095 ins_pipe(ialu_reg_shift);
10096 %}
10097
10098 // This pattern is automatically generated from aarch64_ad.m4
10099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10100 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10101 immL0 zero, iRegL src1, immI src2) %{
10102 match(Set dst (SubL zero (LShiftL src1 src2)));
10103
10104 ins_cost(1.9 * INSN_COST);
10105 format %{ "neg $dst, $src1, LSL $src2" %}
10106
10107 ins_encode %{
10108 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10109 Assembler::LSL, $src2$$constant & 0x3f);
10110 %}
10111
10112 ins_pipe(ialu_reg_shift);
10113 %}
10114
10115 // This pattern is automatically generated from aarch64_ad.m4
10116 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10117 instruct AndI_reg_not_reg(iRegINoSp dst,
10118 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10119 match(Set dst (AndI src1 (XorI src2 m1)));
10120 ins_cost(INSN_COST);
10121 format %{ "bicw $dst, $src1, $src2" %}
10122
10123 ins_encode %{
10124 __ bicw(as_Register($dst$$reg),
10125 as_Register($src1$$reg),
10126 as_Register($src2$$reg),
10127 Assembler::LSL, 0);
10128 %}
10129
10130 ins_pipe(ialu_reg_reg);
10131 %}
10132
10133 // This pattern is automatically generated from aarch64_ad.m4
10134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10135 instruct AndL_reg_not_reg(iRegLNoSp dst,
10136 iRegL src1, iRegL src2, immL_M1 m1) %{
10137 match(Set dst (AndL src1 (XorL src2 m1)));
10138 ins_cost(INSN_COST);
10139 format %{ "bic $dst, $src1, $src2" %}
10140
10141 ins_encode %{
10142 __ bic(as_Register($dst$$reg),
10143 as_Register($src1$$reg),
10144 as_Register($src2$$reg),
10145 Assembler::LSL, 0);
10146 %}
10147
10148 ins_pipe(ialu_reg_reg);
10149 %}
10150
10151 // This pattern is automatically generated from aarch64_ad.m4
10152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10153 instruct OrI_reg_not_reg(iRegINoSp dst,
10154 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10155 match(Set dst (OrI src1 (XorI src2 m1)));
10156 ins_cost(INSN_COST);
10157 format %{ "ornw $dst, $src1, $src2" %}
10158
10159 ins_encode %{
10160 __ ornw(as_Register($dst$$reg),
10161 as_Register($src1$$reg),
10162 as_Register($src2$$reg),
10163 Assembler::LSL, 0);
10164 %}
10165
10166 ins_pipe(ialu_reg_reg);
10167 %}
10168
10169 // This pattern is automatically generated from aarch64_ad.m4
10170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10171 instruct OrL_reg_not_reg(iRegLNoSp dst,
10172 iRegL src1, iRegL src2, immL_M1 m1) %{
10173 match(Set dst (OrL src1 (XorL src2 m1)));
10174 ins_cost(INSN_COST);
10175 format %{ "orn $dst, $src1, $src2" %}
10176
10177 ins_encode %{
10178 __ orn(as_Register($dst$$reg),
10179 as_Register($src1$$reg),
10180 as_Register($src2$$reg),
10181 Assembler::LSL, 0);
10182 %}
10183
10184 ins_pipe(ialu_reg_reg);
10185 %}
10186
10187 // This pattern is automatically generated from aarch64_ad.m4
10188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10189 instruct XorI_reg_not_reg(iRegINoSp dst,
10190 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10191 match(Set dst (XorI m1 (XorI src2 src1)));
10192 ins_cost(INSN_COST);
10193 format %{ "eonw $dst, $src1, $src2" %}
10194
10195 ins_encode %{
10196 __ eonw(as_Register($dst$$reg),
10197 as_Register($src1$$reg),
10198 as_Register($src2$$reg),
10199 Assembler::LSL, 0);
10200 %}
10201
10202 ins_pipe(ialu_reg_reg);
10203 %}
10204
10205 // This pattern is automatically generated from aarch64_ad.m4
10206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10207 instruct XorL_reg_not_reg(iRegLNoSp dst,
10208 iRegL src1, iRegL src2, immL_M1 m1) %{
10209 match(Set dst (XorL m1 (XorL src2 src1)));
10210 ins_cost(INSN_COST);
10211 format %{ "eon $dst, $src1, $src2" %}
10212
10213 ins_encode %{
10214 __ eon(as_Register($dst$$reg),
10215 as_Register($src1$$reg),
10216 as_Register($src2$$reg),
10217 Assembler::LSL, 0);
10218 %}
10219
10220 ins_pipe(ialu_reg_reg);
10221 %}
10222
10223 // This pattern is automatically generated from aarch64_ad.m4
10224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10225 // val & (-1 ^ (val >>> shift)) ==> bicw
10226 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10227 iRegIorL2I src1, iRegIorL2I src2,
10228 immI src3, immI_M1 src4) %{
10229 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10230 ins_cost(1.9 * INSN_COST);
10231 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10232
10233 ins_encode %{
10234 __ bicw(as_Register($dst$$reg),
10235 as_Register($src1$$reg),
10236 as_Register($src2$$reg),
10237 Assembler::LSR,
10238 $src3$$constant & 0x1f);
10239 %}
10240
10241 ins_pipe(ialu_reg_reg_shift);
10242 %}
10243
10244 // This pattern is automatically generated from aarch64_ad.m4
10245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10246 // val & (-1 ^ (val >>> shift)) ==> bic
10247 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10248 iRegL src1, iRegL src2,
10249 immI src3, immL_M1 src4) %{
10250 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10251 ins_cost(1.9 * INSN_COST);
10252 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10253
10254 ins_encode %{
10255 __ bic(as_Register($dst$$reg),
10256 as_Register($src1$$reg),
10257 as_Register($src2$$reg),
10258 Assembler::LSR,
10259 $src3$$constant & 0x3f);
10260 %}
10261
10262 ins_pipe(ialu_reg_reg_shift);
10263 %}
10264
10265 // This pattern is automatically generated from aarch64_ad.m4
10266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10267 // val & (-1 ^ (val >> shift)) ==> bicw
10268 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10269 iRegIorL2I src1, iRegIorL2I src2,
10270 immI src3, immI_M1 src4) %{
10271 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10272 ins_cost(1.9 * INSN_COST);
10273 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10274
10275 ins_encode %{
10276 __ bicw(as_Register($dst$$reg),
10277 as_Register($src1$$reg),
10278 as_Register($src2$$reg),
10279 Assembler::ASR,
10280 $src3$$constant & 0x1f);
10281 %}
10282
10283 ins_pipe(ialu_reg_reg_shift);
10284 %}
10285
10286 // This pattern is automatically generated from aarch64_ad.m4
10287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10288 // val & (-1 ^ (val >> shift)) ==> bic
10289 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10290 iRegL src1, iRegL src2,
10291 immI src3, immL_M1 src4) %{
10292 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10293 ins_cost(1.9 * INSN_COST);
10294 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10295
10296 ins_encode %{
10297 __ bic(as_Register($dst$$reg),
10298 as_Register($src1$$reg),
10299 as_Register($src2$$reg),
10300 Assembler::ASR,
10301 $src3$$constant & 0x3f);
10302 %}
10303
10304 ins_pipe(ialu_reg_reg_shift);
10305 %}
10306
10307 // This pattern is automatically generated from aarch64_ad.m4
10308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10309 // val & (-1 ^ (val ror shift)) ==> bicw
10310 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10311 iRegIorL2I src1, iRegIorL2I src2,
10312 immI src3, immI_M1 src4) %{
10313 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10314 ins_cost(1.9 * INSN_COST);
10315 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10316
10317 ins_encode %{
10318 __ bicw(as_Register($dst$$reg),
10319 as_Register($src1$$reg),
10320 as_Register($src2$$reg),
10321 Assembler::ROR,
10322 $src3$$constant & 0x1f);
10323 %}
10324
10325 ins_pipe(ialu_reg_reg_shift);
10326 %}
10327
10328 // This pattern is automatically generated from aarch64_ad.m4
10329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10330 // val & (-1 ^ (val ror shift)) ==> bic
10331 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10332 iRegL src1, iRegL src2,
10333 immI src3, immL_M1 src4) %{
10334 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10335 ins_cost(1.9 * INSN_COST);
10336 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10337
10338 ins_encode %{
10339 __ bic(as_Register($dst$$reg),
10340 as_Register($src1$$reg),
10341 as_Register($src2$$reg),
10342 Assembler::ROR,
10343 $src3$$constant & 0x3f);
10344 %}
10345
10346 ins_pipe(ialu_reg_reg_shift);
10347 %}
10348
10349 // This pattern is automatically generated from aarch64_ad.m4
10350 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10351 // val & (-1 ^ (val << shift)) ==> bicw
10352 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10353 iRegIorL2I src1, iRegIorL2I src2,
10354 immI src3, immI_M1 src4) %{
10355 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10356 ins_cost(1.9 * INSN_COST);
10357 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10358
10359 ins_encode %{
10360 __ bicw(as_Register($dst$$reg),
10361 as_Register($src1$$reg),
10362 as_Register($src2$$reg),
10363 Assembler::LSL,
10364 $src3$$constant & 0x1f);
10365 %}
10366
10367 ins_pipe(ialu_reg_reg_shift);
10368 %}
10369
10370 // This pattern is automatically generated from aarch64_ad.m4
10371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10372 // val & (-1 ^ (val << shift)) ==> bic
10373 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10374 iRegL src1, iRegL src2,
10375 immI src3, immL_M1 src4) %{
10376 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10377 ins_cost(1.9 * INSN_COST);
10378 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10379
10380 ins_encode %{
10381 __ bic(as_Register($dst$$reg),
10382 as_Register($src1$$reg),
10383 as_Register($src2$$reg),
10384 Assembler::LSL,
10385 $src3$$constant & 0x3f);
10386 %}
10387
10388 ins_pipe(ialu_reg_reg_shift);
10389 %}
10390
10391 // This pattern is automatically generated from aarch64_ad.m4
10392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10393 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10394 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10395 iRegIorL2I src1, iRegIorL2I src2,
10396 immI src3, immI_M1 src4) %{
10397 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10398 ins_cost(1.9 * INSN_COST);
10399 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10400
10401 ins_encode %{
10402 __ eonw(as_Register($dst$$reg),
10403 as_Register($src1$$reg),
10404 as_Register($src2$$reg),
10405 Assembler::LSR,
10406 $src3$$constant & 0x1f);
10407 %}
10408
10409 ins_pipe(ialu_reg_reg_shift);
10410 %}
10411
10412 // This pattern is automatically generated from aarch64_ad.m4
10413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10414 // val ^ (-1 ^ (val >>> shift)) ==> eon
10415 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10416 iRegL src1, iRegL src2,
10417 immI src3, immL_M1 src4) %{
10418 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10419 ins_cost(1.9 * INSN_COST);
10420 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10421
10422 ins_encode %{
10423 __ eon(as_Register($dst$$reg),
10424 as_Register($src1$$reg),
10425 as_Register($src2$$reg),
10426 Assembler::LSR,
10427 $src3$$constant & 0x3f);
10428 %}
10429
10430 ins_pipe(ialu_reg_reg_shift);
10431 %}
10432
10433 // This pattern is automatically generated from aarch64_ad.m4
10434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10435 // val ^ (-1 ^ (val >> shift)) ==> eonw
10436 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10437 iRegIorL2I src1, iRegIorL2I src2,
10438 immI src3, immI_M1 src4) %{
10439 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10440 ins_cost(1.9 * INSN_COST);
10441 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10442
10443 ins_encode %{
10444 __ eonw(as_Register($dst$$reg),
10445 as_Register($src1$$reg),
10446 as_Register($src2$$reg),
10447 Assembler::ASR,
10448 $src3$$constant & 0x1f);
10449 %}
10450
10451 ins_pipe(ialu_reg_reg_shift);
10452 %}
10453
10454 // This pattern is automatically generated from aarch64_ad.m4
10455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10456 // val ^ (-1 ^ (val >> shift)) ==> eon
10457 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10458 iRegL src1, iRegL src2,
10459 immI src3, immL_M1 src4) %{
10460 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10461 ins_cost(1.9 * INSN_COST);
10462 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10463
10464 ins_encode %{
10465 __ eon(as_Register($dst$$reg),
10466 as_Register($src1$$reg),
10467 as_Register($src2$$reg),
10468 Assembler::ASR,
10469 $src3$$constant & 0x3f);
10470 %}
10471
10472 ins_pipe(ialu_reg_reg_shift);
10473 %}
10474
10475 // This pattern is automatically generated from aarch64_ad.m4
10476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10477 // val ^ (-1 ^ (val ror shift)) ==> eonw
10478 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10479 iRegIorL2I src1, iRegIorL2I src2,
10480 immI src3, immI_M1 src4) %{
10481 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10482 ins_cost(1.9 * INSN_COST);
10483 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10484
10485 ins_encode %{
10486 __ eonw(as_Register($dst$$reg),
10487 as_Register($src1$$reg),
10488 as_Register($src2$$reg),
10489 Assembler::ROR,
10490 $src3$$constant & 0x1f);
10491 %}
10492
10493 ins_pipe(ialu_reg_reg_shift);
10494 %}
10495
10496 // This pattern is automatically generated from aarch64_ad.m4
10497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10498 // val ^ (-1 ^ (val ror shift)) ==> eon
10499 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10500 iRegL src1, iRegL src2,
10501 immI src3, immL_M1 src4) %{
10502 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10503 ins_cost(1.9 * INSN_COST);
10504 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10505
10506 ins_encode %{
10507 __ eon(as_Register($dst$$reg),
10508 as_Register($src1$$reg),
10509 as_Register($src2$$reg),
10510 Assembler::ROR,
10511 $src3$$constant & 0x3f);
10512 %}
10513
10514 ins_pipe(ialu_reg_reg_shift);
10515 %}
10516
10517 // This pattern is automatically generated from aarch64_ad.m4
10518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10519 // val ^ (-1 ^ (val << shift)) ==> eonw
10520 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10521 iRegIorL2I src1, iRegIorL2I src2,
10522 immI src3, immI_M1 src4) %{
10523 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10524 ins_cost(1.9 * INSN_COST);
10525 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10526
10527 ins_encode %{
10528 __ eonw(as_Register($dst$$reg),
10529 as_Register($src1$$reg),
10530 as_Register($src2$$reg),
10531 Assembler::LSL,
10532 $src3$$constant & 0x1f);
10533 %}
10534
10535 ins_pipe(ialu_reg_reg_shift);
10536 %}
10537
10538 // This pattern is automatically generated from aarch64_ad.m4
10539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10540 // val ^ (-1 ^ (val << shift)) ==> eon
10541 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10542 iRegL src1, iRegL src2,
10543 immI src3, immL_M1 src4) %{
10544 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10545 ins_cost(1.9 * INSN_COST);
10546 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10547
10548 ins_encode %{
10549 __ eon(as_Register($dst$$reg),
10550 as_Register($src1$$reg),
10551 as_Register($src2$$reg),
10552 Assembler::LSL,
10553 $src3$$constant & 0x3f);
10554 %}
10555
10556 ins_pipe(ialu_reg_reg_shift);
10557 %}
10558
10559 // This pattern is automatically generated from aarch64_ad.m4
10560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10561 // val | (-1 ^ (val >>> shift)) ==> ornw
10562 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10563 iRegIorL2I src1, iRegIorL2I src2,
10564 immI src3, immI_M1 src4) %{
10565 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10566 ins_cost(1.9 * INSN_COST);
10567 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10568
10569 ins_encode %{
10570 __ ornw(as_Register($dst$$reg),
10571 as_Register($src1$$reg),
10572 as_Register($src2$$reg),
10573 Assembler::LSR,
10574 $src3$$constant & 0x1f);
10575 %}
10576
10577 ins_pipe(ialu_reg_reg_shift);
10578 %}
10579
10580 // This pattern is automatically generated from aarch64_ad.m4
10581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10582 // val | (-1 ^ (val >>> shift)) ==> orn
10583 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10584 iRegL src1, iRegL src2,
10585 immI src3, immL_M1 src4) %{
10586 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10587 ins_cost(1.9 * INSN_COST);
10588 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10589
10590 ins_encode %{
10591 __ orn(as_Register($dst$$reg),
10592 as_Register($src1$$reg),
10593 as_Register($src2$$reg),
10594 Assembler::LSR,
10595 $src3$$constant & 0x3f);
10596 %}
10597
10598 ins_pipe(ialu_reg_reg_shift);
10599 %}
10600
10601 // This pattern is automatically generated from aarch64_ad.m4
10602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10603 // val | (-1 ^ (val >> shift)) ==> ornw
10604 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10605 iRegIorL2I src1, iRegIorL2I src2,
10606 immI src3, immI_M1 src4) %{
10607 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10608 ins_cost(1.9 * INSN_COST);
10609 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10610
10611 ins_encode %{
10612 __ ornw(as_Register($dst$$reg),
10613 as_Register($src1$$reg),
10614 as_Register($src2$$reg),
10615 Assembler::ASR,
10616 $src3$$constant & 0x1f);
10617 %}
10618
10619 ins_pipe(ialu_reg_reg_shift);
10620 %}
10621
10622 // This pattern is automatically generated from aarch64_ad.m4
10623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10624 // val | (-1 ^ (val >> shift)) ==> orn
10625 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10626 iRegL src1, iRegL src2,
10627 immI src3, immL_M1 src4) %{
10628 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10629 ins_cost(1.9 * INSN_COST);
10630 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10631
10632 ins_encode %{
10633 __ orn(as_Register($dst$$reg),
10634 as_Register($src1$$reg),
10635 as_Register($src2$$reg),
10636 Assembler::ASR,
10637 $src3$$constant & 0x3f);
10638 %}
10639
10640 ins_pipe(ialu_reg_reg_shift);
10641 %}
10642
10643 // This pattern is automatically generated from aarch64_ad.m4
10644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10645 // val | (-1 ^ (val ror shift)) ==> ornw
10646 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10647 iRegIorL2I src1, iRegIorL2I src2,
10648 immI src3, immI_M1 src4) %{
10649 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10650 ins_cost(1.9 * INSN_COST);
10651 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10652
10653 ins_encode %{
10654 __ ornw(as_Register($dst$$reg),
10655 as_Register($src1$$reg),
10656 as_Register($src2$$reg),
10657 Assembler::ROR,
10658 $src3$$constant & 0x1f);
10659 %}
10660
10661 ins_pipe(ialu_reg_reg_shift);
10662 %}
10663
10664 // This pattern is automatically generated from aarch64_ad.m4
10665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10666 // val | (-1 ^ (val ror shift)) ==> orn
10667 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10668 iRegL src1, iRegL src2,
10669 immI src3, immL_M1 src4) %{
10670 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10671 ins_cost(1.9 * INSN_COST);
10672 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10673
10674 ins_encode %{
10675 __ orn(as_Register($dst$$reg),
10676 as_Register($src1$$reg),
10677 as_Register($src2$$reg),
10678 Assembler::ROR,
10679 $src3$$constant & 0x3f);
10680 %}
10681
10682 ins_pipe(ialu_reg_reg_shift);
10683 %}
10684
10685 // This pattern is automatically generated from aarch64_ad.m4
10686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10687 // val | (-1 ^ (val << shift)) ==> ornw
10688 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10689 iRegIorL2I src1, iRegIorL2I src2,
10690 immI src3, immI_M1 src4) %{
10691 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10692 ins_cost(1.9 * INSN_COST);
10693 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10694
10695 ins_encode %{
10696 __ ornw(as_Register($dst$$reg),
10697 as_Register($src1$$reg),
10698 as_Register($src2$$reg),
10699 Assembler::LSL,
10700 $src3$$constant & 0x1f);
10701 %}
10702
10703 ins_pipe(ialu_reg_reg_shift);
10704 %}
10705
10706 // This pattern is automatically generated from aarch64_ad.m4
10707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10708 // val | (-1 ^ (val << shift)) ==> orn
10709 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10710 iRegL src1, iRegL src2,
10711 immI src3, immL_M1 src4) %{
10712 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10713 ins_cost(1.9 * INSN_COST);
10714 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10715
10716 ins_encode %{
10717 __ orn(as_Register($dst$$reg),
10718 as_Register($src1$$reg),
10719 as_Register($src2$$reg),
10720 Assembler::LSL,
10721 $src3$$constant & 0x3f);
10722 %}
10723
10724 ins_pipe(ialu_reg_reg_shift);
10725 %}
10726
10727 // This pattern is automatically generated from aarch64_ad.m4
10728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10729 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10730 iRegIorL2I src1, iRegIorL2I src2,
10731 immI src3) %{
10732 match(Set dst (AndI src1 (URShiftI src2 src3)));
10733
10734 ins_cost(1.9 * INSN_COST);
10735 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10736
10737 ins_encode %{
10738 __ andw(as_Register($dst$$reg),
10739 as_Register($src1$$reg),
10740 as_Register($src2$$reg),
10741 Assembler::LSR,
10742 $src3$$constant & 0x1f);
10743 %}
10744
10745 ins_pipe(ialu_reg_reg_shift);
10746 %}
10747
10748 // This pattern is automatically generated from aarch64_ad.m4
10749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10750 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10751 iRegL src1, iRegL src2,
10752 immI src3) %{
10753 match(Set dst (AndL src1 (URShiftL src2 src3)));
10754
10755 ins_cost(1.9 * INSN_COST);
10756 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10757
10758 ins_encode %{
10759 __ andr(as_Register($dst$$reg),
10760 as_Register($src1$$reg),
10761 as_Register($src2$$reg),
10762 Assembler::LSR,
10763 $src3$$constant & 0x3f);
10764 %}
10765
10766 ins_pipe(ialu_reg_reg_shift);
10767 %}
10768
10769 // This pattern is automatically generated from aarch64_ad.m4
10770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10771 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10772 iRegIorL2I src1, iRegIorL2I src2,
10773 immI src3) %{
10774 match(Set dst (AndI src1 (RShiftI src2 src3)));
10775
10776 ins_cost(1.9 * INSN_COST);
10777 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10778
10779 ins_encode %{
10780 __ andw(as_Register($dst$$reg),
10781 as_Register($src1$$reg),
10782 as_Register($src2$$reg),
10783 Assembler::ASR,
10784 $src3$$constant & 0x1f);
10785 %}
10786
10787 ins_pipe(ialu_reg_reg_shift);
10788 %}
10789
10790 // This pattern is automatically generated from aarch64_ad.m4
10791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10792 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10793 iRegL src1, iRegL src2,
10794 immI src3) %{
10795 match(Set dst (AndL src1 (RShiftL src2 src3)));
10796
10797 ins_cost(1.9 * INSN_COST);
10798 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10799
10800 ins_encode %{
10801 __ andr(as_Register($dst$$reg),
10802 as_Register($src1$$reg),
10803 as_Register($src2$$reg),
10804 Assembler::ASR,
10805 $src3$$constant & 0x3f);
10806 %}
10807
10808 ins_pipe(ialu_reg_reg_shift);
10809 %}
10810
10811 // This pattern is automatically generated from aarch64_ad.m4
10812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10813 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10814 iRegIorL2I src1, iRegIorL2I src2,
10815 immI src3) %{
10816 match(Set dst (AndI src1 (LShiftI src2 src3)));
10817
10818 ins_cost(1.9 * INSN_COST);
10819 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10820
10821 ins_encode %{
10822 __ andw(as_Register($dst$$reg),
10823 as_Register($src1$$reg),
10824 as_Register($src2$$reg),
10825 Assembler::LSL,
10826 $src3$$constant & 0x1f);
10827 %}
10828
10829 ins_pipe(ialu_reg_reg_shift);
10830 %}
10831
10832 // This pattern is automatically generated from aarch64_ad.m4
10833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10834 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10835 iRegL src1, iRegL src2,
10836 immI src3) %{
10837 match(Set dst (AndL src1 (LShiftL src2 src3)));
10838
10839 ins_cost(1.9 * INSN_COST);
10840 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10841
10842 ins_encode %{
10843 __ andr(as_Register($dst$$reg),
10844 as_Register($src1$$reg),
10845 as_Register($src2$$reg),
10846 Assembler::LSL,
10847 $src3$$constant & 0x3f);
10848 %}
10849
10850 ins_pipe(ialu_reg_reg_shift);
10851 %}
10852
10853 // This pattern is automatically generated from aarch64_ad.m4
10854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10855 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10856 iRegIorL2I src1, iRegIorL2I src2,
10857 immI src3) %{
10858 match(Set dst (AndI src1 (RotateRight src2 src3)));
10859
10860 ins_cost(1.9 * INSN_COST);
10861 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10862
10863 ins_encode %{
10864 __ andw(as_Register($dst$$reg),
10865 as_Register($src1$$reg),
10866 as_Register($src2$$reg),
10867 Assembler::ROR,
10868 $src3$$constant & 0x1f);
10869 %}
10870
10871 ins_pipe(ialu_reg_reg_shift);
10872 %}
10873
10874 // This pattern is automatically generated from aarch64_ad.m4
10875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10876 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10877 iRegL src1, iRegL src2,
10878 immI src3) %{
10879 match(Set dst (AndL src1 (RotateRight src2 src3)));
10880
10881 ins_cost(1.9 * INSN_COST);
10882 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10883
10884 ins_encode %{
10885 __ andr(as_Register($dst$$reg),
10886 as_Register($src1$$reg),
10887 as_Register($src2$$reg),
10888 Assembler::ROR,
10889 $src3$$constant & 0x3f);
10890 %}
10891
10892 ins_pipe(ialu_reg_reg_shift);
10893 %}
10894
10895 // This pattern is automatically generated from aarch64_ad.m4
10896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10897 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10898 iRegIorL2I src1, iRegIorL2I src2,
10899 immI src3) %{
10900 match(Set dst (XorI src1 (URShiftI src2 src3)));
10901
10902 ins_cost(1.9 * INSN_COST);
10903 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10904
10905 ins_encode %{
10906 __ eorw(as_Register($dst$$reg),
10907 as_Register($src1$$reg),
10908 as_Register($src2$$reg),
10909 Assembler::LSR,
10910 $src3$$constant & 0x1f);
10911 %}
10912
10913 ins_pipe(ialu_reg_reg_shift);
10914 %}
10915
10916 // This pattern is automatically generated from aarch64_ad.m4
10917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10918 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10919 iRegL src1, iRegL src2,
10920 immI src3) %{
10921 match(Set dst (XorL src1 (URShiftL src2 src3)));
10922
10923 ins_cost(1.9 * INSN_COST);
10924 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10925
10926 ins_encode %{
10927 __ eor(as_Register($dst$$reg),
10928 as_Register($src1$$reg),
10929 as_Register($src2$$reg),
10930 Assembler::LSR,
10931 $src3$$constant & 0x3f);
10932 %}
10933
10934 ins_pipe(ialu_reg_reg_shift);
10935 %}
10936
10937 // This pattern is automatically generated from aarch64_ad.m4
10938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10939 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10940 iRegIorL2I src1, iRegIorL2I src2,
10941 immI src3) %{
10942 match(Set dst (XorI src1 (RShiftI src2 src3)));
10943
10944 ins_cost(1.9 * INSN_COST);
10945 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10946
10947 ins_encode %{
10948 __ eorw(as_Register($dst$$reg),
10949 as_Register($src1$$reg),
10950 as_Register($src2$$reg),
10951 Assembler::ASR,
10952 $src3$$constant & 0x1f);
10953 %}
10954
10955 ins_pipe(ialu_reg_reg_shift);
10956 %}
10957
10958 // This pattern is automatically generated from aarch64_ad.m4
10959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10960 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10961 iRegL src1, iRegL src2,
10962 immI src3) %{
10963 match(Set dst (XorL src1 (RShiftL src2 src3)));
10964
10965 ins_cost(1.9 * INSN_COST);
10966 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10967
10968 ins_encode %{
10969 __ eor(as_Register($dst$$reg),
10970 as_Register($src1$$reg),
10971 as_Register($src2$$reg),
10972 Assembler::ASR,
10973 $src3$$constant & 0x3f);
10974 %}
10975
10976 ins_pipe(ialu_reg_reg_shift);
10977 %}
10978
10979 // This pattern is automatically generated from aarch64_ad.m4
10980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10981 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10982 iRegIorL2I src1, iRegIorL2I src2,
10983 immI src3) %{
10984 match(Set dst (XorI src1 (LShiftI src2 src3)));
10985
10986 ins_cost(1.9 * INSN_COST);
10987 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10988
10989 ins_encode %{
10990 __ eorw(as_Register($dst$$reg),
10991 as_Register($src1$$reg),
10992 as_Register($src2$$reg),
10993 Assembler::LSL,
10994 $src3$$constant & 0x1f);
10995 %}
10996
10997 ins_pipe(ialu_reg_reg_shift);
10998 %}
10999
11000 // This pattern is automatically generated from aarch64_ad.m4
11001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11002 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11003 iRegL src1, iRegL src2,
11004 immI src3) %{
11005 match(Set dst (XorL src1 (LShiftL src2 src3)));
11006
11007 ins_cost(1.9 * INSN_COST);
11008 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11009
11010 ins_encode %{
11011 __ eor(as_Register($dst$$reg),
11012 as_Register($src1$$reg),
11013 as_Register($src2$$reg),
11014 Assembler::LSL,
11015 $src3$$constant & 0x3f);
11016 %}
11017
11018 ins_pipe(ialu_reg_reg_shift);
11019 %}
11020
11021 // This pattern is automatically generated from aarch64_ad.m4
11022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11023 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11024 iRegIorL2I src1, iRegIorL2I src2,
11025 immI src3) %{
11026 match(Set dst (XorI src1 (RotateRight src2 src3)));
11027
11028 ins_cost(1.9 * INSN_COST);
11029 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11030
11031 ins_encode %{
11032 __ eorw(as_Register($dst$$reg),
11033 as_Register($src1$$reg),
11034 as_Register($src2$$reg),
11035 Assembler::ROR,
11036 $src3$$constant & 0x1f);
11037 %}
11038
11039 ins_pipe(ialu_reg_reg_shift);
11040 %}
11041
11042 // This pattern is automatically generated from aarch64_ad.m4
11043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11044 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11045 iRegL src1, iRegL src2,
11046 immI src3) %{
11047 match(Set dst (XorL src1 (RotateRight src2 src3)));
11048
11049 ins_cost(1.9 * INSN_COST);
11050 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11051
11052 ins_encode %{
11053 __ eor(as_Register($dst$$reg),
11054 as_Register($src1$$reg),
11055 as_Register($src2$$reg),
11056 Assembler::ROR,
11057 $src3$$constant & 0x3f);
11058 %}
11059
11060 ins_pipe(ialu_reg_reg_shift);
11061 %}
11062
11063 // This pattern is automatically generated from aarch64_ad.m4
11064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11065 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11066 iRegIorL2I src1, iRegIorL2I src2,
11067 immI src3) %{
11068 match(Set dst (OrI src1 (URShiftI src2 src3)));
11069
11070 ins_cost(1.9 * INSN_COST);
11071 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11072
11073 ins_encode %{
11074 __ orrw(as_Register($dst$$reg),
11075 as_Register($src1$$reg),
11076 as_Register($src2$$reg),
11077 Assembler::LSR,
11078 $src3$$constant & 0x1f);
11079 %}
11080
11081 ins_pipe(ialu_reg_reg_shift);
11082 %}
11083
11084 // This pattern is automatically generated from aarch64_ad.m4
11085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11086 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11087 iRegL src1, iRegL src2,
11088 immI src3) %{
11089 match(Set dst (OrL src1 (URShiftL src2 src3)));
11090
11091 ins_cost(1.9 * INSN_COST);
11092 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11093
11094 ins_encode %{
11095 __ orr(as_Register($dst$$reg),
11096 as_Register($src1$$reg),
11097 as_Register($src2$$reg),
11098 Assembler::LSR,
11099 $src3$$constant & 0x3f);
11100 %}
11101
11102 ins_pipe(ialu_reg_reg_shift);
11103 %}
11104
11105 // This pattern is automatically generated from aarch64_ad.m4
11106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11107 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11108 iRegIorL2I src1, iRegIorL2I src2,
11109 immI src3) %{
11110 match(Set dst (OrI src1 (RShiftI src2 src3)));
11111
11112 ins_cost(1.9 * INSN_COST);
11113 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11114
11115 ins_encode %{
11116 __ orrw(as_Register($dst$$reg),
11117 as_Register($src1$$reg),
11118 as_Register($src2$$reg),
11119 Assembler::ASR,
11120 $src3$$constant & 0x1f);
11121 %}
11122
11123 ins_pipe(ialu_reg_reg_shift);
11124 %}
11125
11126 // This pattern is automatically generated from aarch64_ad.m4
11127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11128 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11129 iRegL src1, iRegL src2,
11130 immI src3) %{
11131 match(Set dst (OrL src1 (RShiftL src2 src3)));
11132
11133 ins_cost(1.9 * INSN_COST);
11134 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11135
11136 ins_encode %{
11137 __ orr(as_Register($dst$$reg),
11138 as_Register($src1$$reg),
11139 as_Register($src2$$reg),
11140 Assembler::ASR,
11141 $src3$$constant & 0x3f);
11142 %}
11143
11144 ins_pipe(ialu_reg_reg_shift);
11145 %}
11146
11147 // This pattern is automatically generated from aarch64_ad.m4
11148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11149 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11150 iRegIorL2I src1, iRegIorL2I src2,
11151 immI src3) %{
11152 match(Set dst (OrI src1 (LShiftI src2 src3)));
11153
11154 ins_cost(1.9 * INSN_COST);
11155 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11156
11157 ins_encode %{
11158 __ orrw(as_Register($dst$$reg),
11159 as_Register($src1$$reg),
11160 as_Register($src2$$reg),
11161 Assembler::LSL,
11162 $src3$$constant & 0x1f);
11163 %}
11164
11165 ins_pipe(ialu_reg_reg_shift);
11166 %}
11167
11168 // This pattern is automatically generated from aarch64_ad.m4
11169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11170 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11171 iRegL src1, iRegL src2,
11172 immI src3) %{
11173 match(Set dst (OrL src1 (LShiftL src2 src3)));
11174
11175 ins_cost(1.9 * INSN_COST);
11176 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11177
11178 ins_encode %{
11179 __ orr(as_Register($dst$$reg),
11180 as_Register($src1$$reg),
11181 as_Register($src2$$reg),
11182 Assembler::LSL,
11183 $src3$$constant & 0x3f);
11184 %}
11185
11186 ins_pipe(ialu_reg_reg_shift);
11187 %}
11188
11189 // This pattern is automatically generated from aarch64_ad.m4
11190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11191 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11192 iRegIorL2I src1, iRegIorL2I src2,
11193 immI src3) %{
11194 match(Set dst (OrI src1 (RotateRight src2 src3)));
11195
11196 ins_cost(1.9 * INSN_COST);
11197 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11198
11199 ins_encode %{
11200 __ orrw(as_Register($dst$$reg),
11201 as_Register($src1$$reg),
11202 as_Register($src2$$reg),
11203 Assembler::ROR,
11204 $src3$$constant & 0x1f);
11205 %}
11206
11207 ins_pipe(ialu_reg_reg_shift);
11208 %}
11209
11210 // This pattern is automatically generated from aarch64_ad.m4
11211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11212 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11213 iRegL src1, iRegL src2,
11214 immI src3) %{
11215 match(Set dst (OrL src1 (RotateRight src2 src3)));
11216
11217 ins_cost(1.9 * INSN_COST);
11218 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11219
11220 ins_encode %{
11221 __ orr(as_Register($dst$$reg),
11222 as_Register($src1$$reg),
11223 as_Register($src2$$reg),
11224 Assembler::ROR,
11225 $src3$$constant & 0x3f);
11226 %}
11227
11228 ins_pipe(ialu_reg_reg_shift);
11229 %}
11230
11231 // This pattern is automatically generated from aarch64_ad.m4
11232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11233 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11234 iRegIorL2I src1, iRegIorL2I src2,
11235 immI src3) %{
11236 match(Set dst (AddI src1 (URShiftI src2 src3)));
11237
11238 ins_cost(1.9 * INSN_COST);
11239 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11240
11241 ins_encode %{
11242 __ addw(as_Register($dst$$reg),
11243 as_Register($src1$$reg),
11244 as_Register($src2$$reg),
11245 Assembler::LSR,
11246 $src3$$constant & 0x1f);
11247 %}
11248
11249 ins_pipe(ialu_reg_reg_shift);
11250 %}
11251
11252 // This pattern is automatically generated from aarch64_ad.m4
11253 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11254 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11255 iRegL src1, iRegL src2,
11256 immI src3) %{
11257 match(Set dst (AddL src1 (URShiftL src2 src3)));
11258
11259 ins_cost(1.9 * INSN_COST);
11260 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11261
11262 ins_encode %{
11263 __ add(as_Register($dst$$reg),
11264 as_Register($src1$$reg),
11265 as_Register($src2$$reg),
11266 Assembler::LSR,
11267 $src3$$constant & 0x3f);
11268 %}
11269
11270 ins_pipe(ialu_reg_reg_shift);
11271 %}
11272
11273 // This pattern is automatically generated from aarch64_ad.m4
11274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11275 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11276 iRegIorL2I src1, iRegIorL2I src2,
11277 immI src3) %{
11278 match(Set dst (AddI src1 (RShiftI src2 src3)));
11279
11280 ins_cost(1.9 * INSN_COST);
11281 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11282
11283 ins_encode %{
11284 __ addw(as_Register($dst$$reg),
11285 as_Register($src1$$reg),
11286 as_Register($src2$$reg),
11287 Assembler::ASR,
11288 $src3$$constant & 0x1f);
11289 %}
11290
11291 ins_pipe(ialu_reg_reg_shift);
11292 %}
11293
11294 // This pattern is automatically generated from aarch64_ad.m4
11295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11296 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11297 iRegL src1, iRegL src2,
11298 immI src3) %{
11299 match(Set dst (AddL src1 (RShiftL src2 src3)));
11300
11301 ins_cost(1.9 * INSN_COST);
11302 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11303
11304 ins_encode %{
11305 __ add(as_Register($dst$$reg),
11306 as_Register($src1$$reg),
11307 as_Register($src2$$reg),
11308 Assembler::ASR,
11309 $src3$$constant & 0x3f);
11310 %}
11311
11312 ins_pipe(ialu_reg_reg_shift);
11313 %}
11314
11315 // This pattern is automatically generated from aarch64_ad.m4
11316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11317 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11318 iRegIorL2I src1, iRegIorL2I src2,
11319 immI src3) %{
11320 match(Set dst (AddI src1 (LShiftI src2 src3)));
11321
11322 ins_cost(1.9 * INSN_COST);
11323 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11324
11325 ins_encode %{
11326 __ addw(as_Register($dst$$reg),
11327 as_Register($src1$$reg),
11328 as_Register($src2$$reg),
11329 Assembler::LSL,
11330 $src3$$constant & 0x1f);
11331 %}
11332
11333 ins_pipe(ialu_reg_reg_shift);
11334 %}
11335
11336 // This pattern is automatically generated from aarch64_ad.m4
11337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11338 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11339 iRegL src1, iRegL src2,
11340 immI src3) %{
11341 match(Set dst (AddL src1 (LShiftL src2 src3)));
11342
11343 ins_cost(1.9 * INSN_COST);
11344 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11345
11346 ins_encode %{
11347 __ add(as_Register($dst$$reg),
11348 as_Register($src1$$reg),
11349 as_Register($src2$$reg),
11350 Assembler::LSL,
11351 $src3$$constant & 0x3f);
11352 %}
11353
11354 ins_pipe(ialu_reg_reg_shift);
11355 %}
11356
11357 // This pattern is automatically generated from aarch64_ad.m4
11358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11359 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11360 iRegIorL2I src1, iRegIorL2I src2,
11361 immI src3) %{
11362 match(Set dst (SubI src1 (URShiftI src2 src3)));
11363
11364 ins_cost(1.9 * INSN_COST);
11365 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11366
11367 ins_encode %{
11368 __ subw(as_Register($dst$$reg),
11369 as_Register($src1$$reg),
11370 as_Register($src2$$reg),
11371 Assembler::LSR,
11372 $src3$$constant & 0x1f);
11373 %}
11374
11375 ins_pipe(ialu_reg_reg_shift);
11376 %}
11377
11378 // This pattern is automatically generated from aarch64_ad.m4
11379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11380 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11381 iRegL src1, iRegL src2,
11382 immI src3) %{
11383 match(Set dst (SubL src1 (URShiftL src2 src3)));
11384
11385 ins_cost(1.9 * INSN_COST);
11386 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11387
11388 ins_encode %{
11389 __ sub(as_Register($dst$$reg),
11390 as_Register($src1$$reg),
11391 as_Register($src2$$reg),
11392 Assembler::LSR,
11393 $src3$$constant & 0x3f);
11394 %}
11395
11396 ins_pipe(ialu_reg_reg_shift);
11397 %}
11398
11399 // This pattern is automatically generated from aarch64_ad.m4
11400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11401 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11402 iRegIorL2I src1, iRegIorL2I src2,
11403 immI src3) %{
11404 match(Set dst (SubI src1 (RShiftI src2 src3)));
11405
11406 ins_cost(1.9 * INSN_COST);
11407 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11408
11409 ins_encode %{
11410 __ subw(as_Register($dst$$reg),
11411 as_Register($src1$$reg),
11412 as_Register($src2$$reg),
11413 Assembler::ASR,
11414 $src3$$constant & 0x1f);
11415 %}
11416
11417 ins_pipe(ialu_reg_reg_shift);
11418 %}
11419
11420 // This pattern is automatically generated from aarch64_ad.m4
11421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11422 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11423 iRegL src1, iRegL src2,
11424 immI src3) %{
11425 match(Set dst (SubL src1 (RShiftL src2 src3)));
11426
11427 ins_cost(1.9 * INSN_COST);
11428 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11429
11430 ins_encode %{
11431 __ sub(as_Register($dst$$reg),
11432 as_Register($src1$$reg),
11433 as_Register($src2$$reg),
11434 Assembler::ASR,
11435 $src3$$constant & 0x3f);
11436 %}
11437
11438 ins_pipe(ialu_reg_reg_shift);
11439 %}
11440
11441 // This pattern is automatically generated from aarch64_ad.m4
11442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11443 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11444 iRegIorL2I src1, iRegIorL2I src2,
11445 immI src3) %{
11446 match(Set dst (SubI src1 (LShiftI src2 src3)));
11447
11448 ins_cost(1.9 * INSN_COST);
11449 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11450
11451 ins_encode %{
11452 __ subw(as_Register($dst$$reg),
11453 as_Register($src1$$reg),
11454 as_Register($src2$$reg),
11455 Assembler::LSL,
11456 $src3$$constant & 0x1f);
11457 %}
11458
11459 ins_pipe(ialu_reg_reg_shift);
11460 %}
11461
11462 // This pattern is automatically generated from aarch64_ad.m4
11463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11464 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11465 iRegL src1, iRegL src2,
11466 immI src3) %{
11467 match(Set dst (SubL src1 (LShiftL src2 src3)));
11468
11469 ins_cost(1.9 * INSN_COST);
11470 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11471
11472 ins_encode %{
11473 __ sub(as_Register($dst$$reg),
11474 as_Register($src1$$reg),
11475 as_Register($src2$$reg),
11476 Assembler::LSL,
11477 $src3$$constant & 0x3f);
11478 %}
11479
11480 ins_pipe(ialu_reg_reg_shift);
11481 %}
11482
11483 // This pattern is automatically generated from aarch64_ad.m4
11484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11485
11486 // Shift Left followed by Shift Right.
11487 // This idiom is used by the compiler for the i2b bytecode etc.
11488 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11489 %{
11490 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11491 ins_cost(INSN_COST * 2);
11492 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11493 ins_encode %{
11494 int lshift = $lshift_count$$constant & 63;
11495 int rshift = $rshift_count$$constant & 63;
11496 int s = 63 - lshift;
11497 int r = (rshift - lshift) & 63;
11498 __ sbfm(as_Register($dst$$reg),
11499 as_Register($src$$reg),
11500 r, s);
11501 %}
11502
11503 ins_pipe(ialu_reg_shift);
11504 %}
11505
11506 // This pattern is automatically generated from aarch64_ad.m4
11507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11508
11509 // Shift Left followed by Shift Right.
11510 // This idiom is used by the compiler for the i2b bytecode etc.
11511 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11512 %{
11513 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11514 ins_cost(INSN_COST * 2);
11515 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11516 ins_encode %{
11517 int lshift = $lshift_count$$constant & 31;
11518 int rshift = $rshift_count$$constant & 31;
11519 int s = 31 - lshift;
11520 int r = (rshift - lshift) & 31;
11521 __ sbfmw(as_Register($dst$$reg),
11522 as_Register($src$$reg),
11523 r, s);
11524 %}
11525
11526 ins_pipe(ialu_reg_shift);
11527 %}
11528
11529 // This pattern is automatically generated from aarch64_ad.m4
11530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11531
11532 // Shift Left followed by Shift Right.
11533 // This idiom is used by the compiler for the i2b bytecode etc.
11534 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11535 %{
11536 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11537 ins_cost(INSN_COST * 2);
11538 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11539 ins_encode %{
11540 int lshift = $lshift_count$$constant & 63;
11541 int rshift = $rshift_count$$constant & 63;
11542 int s = 63 - lshift;
11543 int r = (rshift - lshift) & 63;
11544 __ ubfm(as_Register($dst$$reg),
11545 as_Register($src$$reg),
11546 r, s);
11547 %}
11548
11549 ins_pipe(ialu_reg_shift);
11550 %}
11551
11552 // This pattern is automatically generated from aarch64_ad.m4
11553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11554
11555 // Shift Left followed by Shift Right.
11556 // This idiom is used by the compiler for the i2b bytecode etc.
11557 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11558 %{
11559 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11560 ins_cost(INSN_COST * 2);
11561 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11562 ins_encode %{
11563 int lshift = $lshift_count$$constant & 31;
11564 int rshift = $rshift_count$$constant & 31;
11565 int s = 31 - lshift;
11566 int r = (rshift - lshift) & 31;
11567 __ ubfmw(as_Register($dst$$reg),
11568 as_Register($src$$reg),
11569 r, s);
11570 %}
11571
11572 ins_pipe(ialu_reg_shift);
11573 %}
11574
11575 // Bitfield extract with shift & mask
11576
11577 // This pattern is automatically generated from aarch64_ad.m4
11578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11579 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11580 %{
11581 match(Set dst (AndI (URShiftI src rshift) mask));
11582 // Make sure we are not going to exceed what ubfxw can do.
11583 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11584
11585 ins_cost(INSN_COST);
11586 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11587 ins_encode %{
11588 int rshift = $rshift$$constant & 31;
11589 intptr_t mask = $mask$$constant;
11590 int width = exact_log2(mask+1);
11591 __ ubfxw(as_Register($dst$$reg),
11592 as_Register($src$$reg), rshift, width);
11593 %}
11594 ins_pipe(ialu_reg_shift);
11595 %}
11596
11597 // This pattern is automatically generated from aarch64_ad.m4
11598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11599 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11600 %{
11601 match(Set dst (AndL (URShiftL src rshift) mask));
11602 // Make sure we are not going to exceed what ubfx can do.
11603 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11604
11605 ins_cost(INSN_COST);
11606 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11607 ins_encode %{
11608 int rshift = $rshift$$constant & 63;
11609 intptr_t mask = $mask$$constant;
11610 int width = exact_log2_long(mask+1);
11611 __ ubfx(as_Register($dst$$reg),
11612 as_Register($src$$reg), rshift, width);
11613 %}
11614 ins_pipe(ialu_reg_shift);
11615 %}
11616
11617
11618 // This pattern is automatically generated from aarch64_ad.m4
11619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11620
11621 // We can use ubfx when extending an And with a mask when we know mask
11622 // is positive. We know that because immI_bitmask guarantees it.
11623 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11624 %{
11625 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11626 // Make sure we are not going to exceed what ubfxw can do.
11627 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11628
11629 ins_cost(INSN_COST * 2);
11630 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11631 ins_encode %{
11632 int rshift = $rshift$$constant & 31;
11633 intptr_t mask = $mask$$constant;
11634 int width = exact_log2(mask+1);
11635 __ ubfx(as_Register($dst$$reg),
11636 as_Register($src$$reg), rshift, width);
11637 %}
11638 ins_pipe(ialu_reg_shift);
11639 %}
11640
11641
11642 // This pattern is automatically generated from aarch64_ad.m4
11643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11644
11645 // We can use ubfiz when masking by a positive number and then left shifting the result.
11646 // We know that the mask is positive because immI_bitmask guarantees it.
11647 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11648 %{
11649 match(Set dst (LShiftI (AndI src mask) lshift));
11650 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11651
11652 ins_cost(INSN_COST);
11653 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11654 ins_encode %{
11655 int lshift = $lshift$$constant & 31;
11656 intptr_t mask = $mask$$constant;
11657 int width = exact_log2(mask+1);
11658 __ ubfizw(as_Register($dst$$reg),
11659 as_Register($src$$reg), lshift, width);
11660 %}
11661 ins_pipe(ialu_reg_shift);
11662 %}
11663
11664 // This pattern is automatically generated from aarch64_ad.m4
11665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11666
11667 // We can use ubfiz when masking by a positive number and then left shifting the result.
11668 // We know that the mask is positive because immL_bitmask guarantees it.
11669 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11670 %{
11671 match(Set dst (LShiftL (AndL src mask) lshift));
11672 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11673
11674 ins_cost(INSN_COST);
11675 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11676 ins_encode %{
11677 int lshift = $lshift$$constant & 63;
11678 intptr_t mask = $mask$$constant;
11679 int width = exact_log2_long(mask+1);
11680 __ ubfiz(as_Register($dst$$reg),
11681 as_Register($src$$reg), lshift, width);
11682 %}
11683 ins_pipe(ialu_reg_shift);
11684 %}
11685
11686 // This pattern is automatically generated from aarch64_ad.m4
11687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11688
11689 // We can use ubfiz when masking by a positive number and then left shifting the result.
11690 // We know that the mask is positive because immI_bitmask guarantees it.
11691 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11692 %{
11693 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11694 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11695
11696 ins_cost(INSN_COST);
11697 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11698 ins_encode %{
11699 int lshift = $lshift$$constant & 31;
11700 intptr_t mask = $mask$$constant;
11701 int width = exact_log2(mask+1);
11702 __ ubfizw(as_Register($dst$$reg),
11703 as_Register($src$$reg), lshift, width);
11704 %}
11705 ins_pipe(ialu_reg_shift);
11706 %}
11707
11708 // This pattern is automatically generated from aarch64_ad.m4
11709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11710
11711 // We can use ubfiz when masking by a positive number and then left shifting the result.
11712 // We know that the mask is positive because immL_bitmask guarantees it.
11713 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11714 %{
11715 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11716 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11717
11718 ins_cost(INSN_COST);
11719 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11720 ins_encode %{
11721 int lshift = $lshift$$constant & 63;
11722 intptr_t mask = $mask$$constant;
11723 int width = exact_log2_long(mask+1);
11724 __ ubfiz(as_Register($dst$$reg),
11725 as_Register($src$$reg), lshift, width);
11726 %}
11727 ins_pipe(ialu_reg_shift);
11728 %}
11729
11730
11731 // This pattern is automatically generated from aarch64_ad.m4
11732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11733
11734 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11735 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11736 %{
11737 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11738 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11739
11740 ins_cost(INSN_COST);
11741 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11742 ins_encode %{
11743 int lshift = $lshift$$constant & 63;
11744 intptr_t mask = $mask$$constant;
11745 int width = exact_log2(mask+1);
11746 __ ubfiz(as_Register($dst$$reg),
11747 as_Register($src$$reg), lshift, width);
11748 %}
11749 ins_pipe(ialu_reg_shift);
11750 %}
11751
11752 // This pattern is automatically generated from aarch64_ad.m4
11753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11754
11755 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11756 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11757 %{
11758 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11759 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11760
11761 ins_cost(INSN_COST);
11762 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11763 ins_encode %{
11764 int lshift = $lshift$$constant & 31;
11765 intptr_t mask = $mask$$constant;
11766 int width = exact_log2(mask+1);
11767 __ ubfiz(as_Register($dst$$reg),
11768 as_Register($src$$reg), lshift, width);
11769 %}
11770 ins_pipe(ialu_reg_shift);
11771 %}
11772
11773 // This pattern is automatically generated from aarch64_ad.m4
11774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11775
11776 // Can skip int2long conversions after AND with small bitmask
11777 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11778 %{
11779 match(Set dst (ConvI2L (AndI src msk)));
11780 ins_cost(INSN_COST);
11781 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11782 ins_encode %{
11783 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11784 %}
11785 ins_pipe(ialu_reg_shift);
11786 %}
11787
11788
11789 // Rotations
11790
11791 // This pattern is automatically generated from aarch64_ad.m4
11792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11793 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11794 %{
11795 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11796 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11797
11798 ins_cost(INSN_COST);
11799 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11800
11801 ins_encode %{
11802 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11803 $rshift$$constant & 63);
11804 %}
11805 ins_pipe(ialu_reg_reg_extr);
11806 %}
11807
11808
11809 // This pattern is automatically generated from aarch64_ad.m4
11810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11811 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11812 %{
11813 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11814 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11815
11816 ins_cost(INSN_COST);
11817 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11818
11819 ins_encode %{
11820 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11821 $rshift$$constant & 31);
11822 %}
11823 ins_pipe(ialu_reg_reg_extr);
11824 %}
11825
11826
11827 // This pattern is automatically generated from aarch64_ad.m4
11828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11829 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11830 %{
11831 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11832 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11833
11834 ins_cost(INSN_COST);
11835 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11836
11837 ins_encode %{
11838 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11839 $rshift$$constant & 63);
11840 %}
11841 ins_pipe(ialu_reg_reg_extr);
11842 %}
11843
11844
11845 // This pattern is automatically generated from aarch64_ad.m4
11846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11847 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11848 %{
11849 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11850 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11851
11852 ins_cost(INSN_COST);
11853 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11854
11855 ins_encode %{
11856 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11857 $rshift$$constant & 31);
11858 %}
11859 ins_pipe(ialu_reg_reg_extr);
11860 %}
11861
11862 // This pattern is automatically generated from aarch64_ad.m4
11863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11864 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11865 %{
11866 match(Set dst (RotateRight src shift));
11867
11868 ins_cost(INSN_COST);
11869 format %{ "ror $dst, $src, $shift" %}
11870
11871 ins_encode %{
11872 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11873 $shift$$constant & 0x1f);
11874 %}
11875 ins_pipe(ialu_reg_reg_vshift);
11876 %}
11877
11878 // This pattern is automatically generated from aarch64_ad.m4
11879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11880 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11881 %{
11882 match(Set dst (RotateRight src shift));
11883
11884 ins_cost(INSN_COST);
11885 format %{ "ror $dst, $src, $shift" %}
11886
11887 ins_encode %{
11888 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11889 $shift$$constant & 0x3f);
11890 %}
11891 ins_pipe(ialu_reg_reg_vshift);
11892 %}
11893
11894 // This pattern is automatically generated from aarch64_ad.m4
11895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11896 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11897 %{
11898 match(Set dst (RotateRight src shift));
11899
11900 ins_cost(INSN_COST);
11901 format %{ "ror $dst, $src, $shift" %}
11902
11903 ins_encode %{
11904 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11905 %}
11906 ins_pipe(ialu_reg_reg_vshift);
11907 %}
11908
11909 // This pattern is automatically generated from aarch64_ad.m4
11910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11911 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11912 %{
11913 match(Set dst (RotateRight src shift));
11914
11915 ins_cost(INSN_COST);
11916 format %{ "ror $dst, $src, $shift" %}
11917
11918 ins_encode %{
11919 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11920 %}
11921 ins_pipe(ialu_reg_reg_vshift);
11922 %}
11923
11924 // This pattern is automatically generated from aarch64_ad.m4
11925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11926 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11927 %{
11928 match(Set dst (RotateLeft src shift));
11929
11930 ins_cost(INSN_COST);
11931 format %{ "rol $dst, $src, $shift" %}
11932
11933 ins_encode %{
11934 __ subw(rscratch1, zr, as_Register($shift$$reg));
11935 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11936 %}
11937 ins_pipe(ialu_reg_reg_vshift);
11938 %}
11939
11940 // This pattern is automatically generated from aarch64_ad.m4
11941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11942 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11943 %{
11944 match(Set dst (RotateLeft src shift));
11945
11946 ins_cost(INSN_COST);
11947 format %{ "rol $dst, $src, $shift" %}
11948
11949 ins_encode %{
11950 __ subw(rscratch1, zr, as_Register($shift$$reg));
11951 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11952 %}
11953 ins_pipe(ialu_reg_reg_vshift);
11954 %}
11955
11956
11957 // Add/subtract (extended)
11958
11959 // This pattern is automatically generated from aarch64_ad.m4
11960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11961 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11962 %{
11963 match(Set dst (AddL src1 (ConvI2L src2)));
11964 ins_cost(INSN_COST);
11965 format %{ "add $dst, $src1, $src2, sxtw" %}
11966
11967 ins_encode %{
11968 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11969 as_Register($src2$$reg), ext::sxtw);
11970 %}
11971 ins_pipe(ialu_reg_reg);
11972 %}
11973
11974 // This pattern is automatically generated from aarch64_ad.m4
11975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11976 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11977 %{
11978 match(Set dst (SubL src1 (ConvI2L src2)));
11979 ins_cost(INSN_COST);
11980 format %{ "sub $dst, $src1, $src2, sxtw" %}
11981
11982 ins_encode %{
11983 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11984 as_Register($src2$$reg), ext::sxtw);
11985 %}
11986 ins_pipe(ialu_reg_reg);
11987 %}
11988
11989 // This pattern is automatically generated from aarch64_ad.m4
11990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11991 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11992 %{
11993 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11994 ins_cost(INSN_COST);
11995 format %{ "add $dst, $src1, $src2, sxth" %}
11996
11997 ins_encode %{
11998 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11999 as_Register($src2$$reg), ext::sxth);
12000 %}
12001 ins_pipe(ialu_reg_reg);
12002 %}
12003
12004 // This pattern is automatically generated from aarch64_ad.m4
12005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12006 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12007 %{
12008 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12009 ins_cost(INSN_COST);
12010 format %{ "add $dst, $src1, $src2, sxtb" %}
12011
12012 ins_encode %{
12013 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12014 as_Register($src2$$reg), ext::sxtb);
12015 %}
12016 ins_pipe(ialu_reg_reg);
12017 %}
12018
12019 // This pattern is automatically generated from aarch64_ad.m4
12020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12021 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12022 %{
12023 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12024 ins_cost(INSN_COST);
12025 format %{ "add $dst, $src1, $src2, uxtb" %}
12026
12027 ins_encode %{
12028 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12029 as_Register($src2$$reg), ext::uxtb);
12030 %}
12031 ins_pipe(ialu_reg_reg);
12032 %}
12033
12034 // This pattern is automatically generated from aarch64_ad.m4
12035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12036 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12037 %{
12038 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12039 ins_cost(INSN_COST);
12040 format %{ "add $dst, $src1, $src2, sxth" %}
12041
12042 ins_encode %{
12043 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12044 as_Register($src2$$reg), ext::sxth);
12045 %}
12046 ins_pipe(ialu_reg_reg);
12047 %}
12048
12049 // This pattern is automatically generated from aarch64_ad.m4
12050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12051 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12052 %{
12053 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12054 ins_cost(INSN_COST);
12055 format %{ "add $dst, $src1, $src2, sxtw" %}
12056
12057 ins_encode %{
12058 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12059 as_Register($src2$$reg), ext::sxtw);
12060 %}
12061 ins_pipe(ialu_reg_reg);
12062 %}
12063
12064 // This pattern is automatically generated from aarch64_ad.m4
12065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12066 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12067 %{
12068 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12069 ins_cost(INSN_COST);
12070 format %{ "add $dst, $src1, $src2, sxtb" %}
12071
12072 ins_encode %{
12073 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12074 as_Register($src2$$reg), ext::sxtb);
12075 %}
12076 ins_pipe(ialu_reg_reg);
12077 %}
12078
12079 // This pattern is automatically generated from aarch64_ad.m4
12080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12081 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12082 %{
12083 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12084 ins_cost(INSN_COST);
12085 format %{ "add $dst, $src1, $src2, uxtb" %}
12086
12087 ins_encode %{
12088 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12089 as_Register($src2$$reg), ext::uxtb);
12090 %}
12091 ins_pipe(ialu_reg_reg);
12092 %}
12093
12094 // This pattern is automatically generated from aarch64_ad.m4
12095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12096 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12097 %{
12098 match(Set dst (AddI src1 (AndI src2 mask)));
12099 ins_cost(INSN_COST);
12100 format %{ "addw $dst, $src1, $src2, uxtb" %}
12101
12102 ins_encode %{
12103 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12104 as_Register($src2$$reg), ext::uxtb);
12105 %}
12106 ins_pipe(ialu_reg_reg);
12107 %}
12108
12109 // This pattern is automatically generated from aarch64_ad.m4
12110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12111 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12112 %{
12113 match(Set dst (AddI src1 (AndI src2 mask)));
12114 ins_cost(INSN_COST);
12115 format %{ "addw $dst, $src1, $src2, uxth" %}
12116
12117 ins_encode %{
12118 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12119 as_Register($src2$$reg), ext::uxth);
12120 %}
12121 ins_pipe(ialu_reg_reg);
12122 %}
12123
12124 // This pattern is automatically generated from aarch64_ad.m4
12125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12126 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12127 %{
12128 match(Set dst (AddL src1 (AndL src2 mask)));
12129 ins_cost(INSN_COST);
12130 format %{ "add $dst, $src1, $src2, uxtb" %}
12131
12132 ins_encode %{
12133 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12134 as_Register($src2$$reg), ext::uxtb);
12135 %}
12136 ins_pipe(ialu_reg_reg);
12137 %}
12138
12139 // This pattern is automatically generated from aarch64_ad.m4
12140 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12141 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12142 %{
12143 match(Set dst (AddL src1 (AndL src2 mask)));
12144 ins_cost(INSN_COST);
12145 format %{ "add $dst, $src1, $src2, uxth" %}
12146
12147 ins_encode %{
12148 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12149 as_Register($src2$$reg), ext::uxth);
12150 %}
12151 ins_pipe(ialu_reg_reg);
12152 %}
12153
12154 // This pattern is automatically generated from aarch64_ad.m4
12155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12156 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12157 %{
12158 match(Set dst (AddL src1 (AndL src2 mask)));
12159 ins_cost(INSN_COST);
12160 format %{ "add $dst, $src1, $src2, uxtw" %}
12161
12162 ins_encode %{
12163 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12164 as_Register($src2$$reg), ext::uxtw);
12165 %}
12166 ins_pipe(ialu_reg_reg);
12167 %}
12168
12169 // This pattern is automatically generated from aarch64_ad.m4
12170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12171 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12172 %{
12173 match(Set dst (SubI src1 (AndI src2 mask)));
12174 ins_cost(INSN_COST);
12175 format %{ "subw $dst, $src1, $src2, uxtb" %}
12176
12177 ins_encode %{
12178 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12179 as_Register($src2$$reg), ext::uxtb);
12180 %}
12181 ins_pipe(ialu_reg_reg);
12182 %}
12183
12184 // This pattern is automatically generated from aarch64_ad.m4
12185 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12186 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12187 %{
12188 match(Set dst (SubI src1 (AndI src2 mask)));
12189 ins_cost(INSN_COST);
12190 format %{ "subw $dst, $src1, $src2, uxth" %}
12191
12192 ins_encode %{
12193 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12194 as_Register($src2$$reg), ext::uxth);
12195 %}
12196 ins_pipe(ialu_reg_reg);
12197 %}
12198
12199 // This pattern is automatically generated from aarch64_ad.m4
12200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12201 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12202 %{
12203 match(Set dst (SubL src1 (AndL src2 mask)));
12204 ins_cost(INSN_COST);
12205 format %{ "sub $dst, $src1, $src2, uxtb" %}
12206
12207 ins_encode %{
12208 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12209 as_Register($src2$$reg), ext::uxtb);
12210 %}
12211 ins_pipe(ialu_reg_reg);
12212 %}
12213
12214 // This pattern is automatically generated from aarch64_ad.m4
12215 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12216 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12217 %{
12218 match(Set dst (SubL src1 (AndL src2 mask)));
12219 ins_cost(INSN_COST);
12220 format %{ "sub $dst, $src1, $src2, uxth" %}
12221
12222 ins_encode %{
12223 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12224 as_Register($src2$$reg), ext::uxth);
12225 %}
12226 ins_pipe(ialu_reg_reg);
12227 %}
12228
12229 // This pattern is automatically generated from aarch64_ad.m4
12230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12231 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12232 %{
12233 match(Set dst (SubL src1 (AndL src2 mask)));
12234 ins_cost(INSN_COST);
12235 format %{ "sub $dst, $src1, $src2, uxtw" %}
12236
12237 ins_encode %{
12238 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12239 as_Register($src2$$reg), ext::uxtw);
12240 %}
12241 ins_pipe(ialu_reg_reg);
12242 %}
12243
12244
12245 // This pattern is automatically generated from aarch64_ad.m4
12246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12247 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12248 %{
12249 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12250 ins_cost(1.9 * INSN_COST);
12251 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12252
12253 ins_encode %{
12254 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12255 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12256 %}
12257 ins_pipe(ialu_reg_reg_shift);
12258 %}
12259
12260 // This pattern is automatically generated from aarch64_ad.m4
12261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12262 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12263 %{
12264 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12265 ins_cost(1.9 * INSN_COST);
12266 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12267
12268 ins_encode %{
12269 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12270 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12271 %}
12272 ins_pipe(ialu_reg_reg_shift);
12273 %}
12274
12275 // This pattern is automatically generated from aarch64_ad.m4
12276 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12277 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12278 %{
12279 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12280 ins_cost(1.9 * INSN_COST);
12281 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12282
12283 ins_encode %{
12284 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12285 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12286 %}
12287 ins_pipe(ialu_reg_reg_shift);
12288 %}
12289
12290 // This pattern is automatically generated from aarch64_ad.m4
12291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12292 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12293 %{
12294 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12295 ins_cost(1.9 * INSN_COST);
12296 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12297
12298 ins_encode %{
12299 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12300 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12301 %}
12302 ins_pipe(ialu_reg_reg_shift);
12303 %}
12304
12305 // This pattern is automatically generated from aarch64_ad.m4
12306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12307 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12308 %{
12309 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12310 ins_cost(1.9 * INSN_COST);
12311 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12312
12313 ins_encode %{
12314 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12315 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12316 %}
12317 ins_pipe(ialu_reg_reg_shift);
12318 %}
12319
12320 // This pattern is automatically generated from aarch64_ad.m4
12321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12322 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12323 %{
12324 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12325 ins_cost(1.9 * INSN_COST);
12326 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12327
12328 ins_encode %{
12329 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12330 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12331 %}
12332 ins_pipe(ialu_reg_reg_shift);
12333 %}
12334
12335 // This pattern is automatically generated from aarch64_ad.m4
12336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12337 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12338 %{
12339 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12340 ins_cost(1.9 * INSN_COST);
12341 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12342
12343 ins_encode %{
12344 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12345 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12346 %}
12347 ins_pipe(ialu_reg_reg_shift);
12348 %}
12349
12350 // This pattern is automatically generated from aarch64_ad.m4
12351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12352 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12353 %{
12354 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12355 ins_cost(1.9 * INSN_COST);
12356 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12357
12358 ins_encode %{
12359 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12360 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12361 %}
12362 ins_pipe(ialu_reg_reg_shift);
12363 %}
12364
12365 // This pattern is automatically generated from aarch64_ad.m4
12366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12367 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12368 %{
12369 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12370 ins_cost(1.9 * INSN_COST);
12371 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12372
12373 ins_encode %{
12374 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12375 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12376 %}
12377 ins_pipe(ialu_reg_reg_shift);
12378 %}
12379
12380 // This pattern is automatically generated from aarch64_ad.m4
12381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12382 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12383 %{
12384 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12385 ins_cost(1.9 * INSN_COST);
12386 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12387
12388 ins_encode %{
12389 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12390 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12391 %}
12392 ins_pipe(ialu_reg_reg_shift);
12393 %}
12394
12395 // This pattern is automatically generated from aarch64_ad.m4
12396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12397 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12398 %{
12399 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12400 ins_cost(1.9 * INSN_COST);
12401 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12402
12403 ins_encode %{
12404 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12405 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12406 %}
12407 ins_pipe(ialu_reg_reg_shift);
12408 %}
12409
12410 // This pattern is automatically generated from aarch64_ad.m4
12411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12412 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12413 %{
12414 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12415 ins_cost(1.9 * INSN_COST);
12416 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12417
12418 ins_encode %{
12419 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12420 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12421 %}
12422 ins_pipe(ialu_reg_reg_shift);
12423 %}
12424
12425 // This pattern is automatically generated from aarch64_ad.m4
12426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12427 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12428 %{
12429 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12430 ins_cost(1.9 * INSN_COST);
12431 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12432
12433 ins_encode %{
12434 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12435 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12436 %}
12437 ins_pipe(ialu_reg_reg_shift);
12438 %}
12439
12440 // This pattern is automatically generated from aarch64_ad.m4
12441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12442 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12443 %{
12444 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12445 ins_cost(1.9 * INSN_COST);
12446 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12447
12448 ins_encode %{
12449 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12450 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12451 %}
12452 ins_pipe(ialu_reg_reg_shift);
12453 %}
12454
12455 // This pattern is automatically generated from aarch64_ad.m4
12456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12457 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12458 %{
12459 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12460 ins_cost(1.9 * INSN_COST);
12461 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12462
12463 ins_encode %{
12464 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12465 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12466 %}
12467 ins_pipe(ialu_reg_reg_shift);
12468 %}
12469
12470 // This pattern is automatically generated from aarch64_ad.m4
12471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12472 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12473 %{
12474 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12475 ins_cost(1.9 * INSN_COST);
12476 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12477
12478 ins_encode %{
12479 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12480 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12481 %}
12482 ins_pipe(ialu_reg_reg_shift);
12483 %}
12484
12485 // This pattern is automatically generated from aarch64_ad.m4
12486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12487 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12488 %{
12489 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12490 ins_cost(1.9 * INSN_COST);
12491 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12492
12493 ins_encode %{
12494 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12495 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12496 %}
12497 ins_pipe(ialu_reg_reg_shift);
12498 %}
12499
12500 // This pattern is automatically generated from aarch64_ad.m4
12501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12502 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12503 %{
12504 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12505 ins_cost(1.9 * INSN_COST);
12506 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12507
12508 ins_encode %{
12509 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12510 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12511 %}
12512 ins_pipe(ialu_reg_reg_shift);
12513 %}
12514
12515 // This pattern is automatically generated from aarch64_ad.m4
12516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12517 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12518 %{
12519 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12520 ins_cost(1.9 * INSN_COST);
12521 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12522
12523 ins_encode %{
12524 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12525 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12526 %}
12527 ins_pipe(ialu_reg_reg_shift);
12528 %}
12529
12530 // This pattern is automatically generated from aarch64_ad.m4
12531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12532 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12533 %{
12534 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12535 ins_cost(1.9 * INSN_COST);
12536 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12537
12538 ins_encode %{
12539 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12540 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12541 %}
12542 ins_pipe(ialu_reg_reg_shift);
12543 %}
12544
12545 // This pattern is automatically generated from aarch64_ad.m4
12546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12547 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12548 %{
12549 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12550 ins_cost(1.9 * INSN_COST);
12551 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12552
12553 ins_encode %{
12554 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12555 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12556 %}
12557 ins_pipe(ialu_reg_reg_shift);
12558 %}
12559
12560 // This pattern is automatically generated from aarch64_ad.m4
12561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12562 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12563 %{
12564 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12565 ins_cost(1.9 * INSN_COST);
12566 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12567
12568 ins_encode %{
12569 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12570 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12571 %}
12572 ins_pipe(ialu_reg_reg_shift);
12573 %}
12574
12575 // This pattern is automatically generated from aarch64_ad.m4
12576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12577 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12578 %{
12579 effect(DEF dst, USE src1, USE src2, USE cr);
12580 ins_cost(INSN_COST * 2);
12581 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12582
12583 ins_encode %{
12584 __ cselw($dst$$Register,
12585 $src1$$Register,
12586 $src2$$Register,
12587 Assembler::LT);
12588 %}
12589 ins_pipe(icond_reg_reg);
12590 %}
12591
12592 // This pattern is automatically generated from aarch64_ad.m4
12593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12594 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12595 %{
12596 effect(DEF dst, USE src1, USE src2, USE cr);
12597 ins_cost(INSN_COST * 2);
12598 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12599
12600 ins_encode %{
12601 __ cselw($dst$$Register,
12602 $src1$$Register,
12603 $src2$$Register,
12604 Assembler::GT);
12605 %}
12606 ins_pipe(icond_reg_reg);
12607 %}
12608
12609 // This pattern is automatically generated from aarch64_ad.m4
12610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12611 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12612 %{
12613 effect(DEF dst, USE src1, USE cr);
12614 ins_cost(INSN_COST * 2);
12615 format %{ "cselw $dst, $src1, zr lt\t" %}
12616
12617 ins_encode %{
12618 __ cselw($dst$$Register,
12619 $src1$$Register,
12620 zr,
12621 Assembler::LT);
12622 %}
12623 ins_pipe(icond_reg);
12624 %}
12625
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12629 %{
12630 effect(DEF dst, USE src1, USE cr);
12631 ins_cost(INSN_COST * 2);
12632 format %{ "cselw $dst, $src1, zr gt\t" %}
12633
12634 ins_encode %{
12635 __ cselw($dst$$Register,
12636 $src1$$Register,
12637 zr,
12638 Assembler::GT);
12639 %}
12640 ins_pipe(icond_reg);
12641 %}
12642
12643 // This pattern is automatically generated from aarch64_ad.m4
12644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12645 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12646 %{
12647 effect(DEF dst, USE src1, USE cr);
12648 ins_cost(INSN_COST * 2);
12649 format %{ "csincw $dst, $src1, zr le\t" %}
12650
12651 ins_encode %{
12652 __ csincw($dst$$Register,
12653 $src1$$Register,
12654 zr,
12655 Assembler::LE);
12656 %}
12657 ins_pipe(icond_reg);
12658 %}
12659
12660 // This pattern is automatically generated from aarch64_ad.m4
12661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12662 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12663 %{
12664 effect(DEF dst, USE src1, USE cr);
12665 ins_cost(INSN_COST * 2);
12666 format %{ "csincw $dst, $src1, zr gt\t" %}
12667
12668 ins_encode %{
12669 __ csincw($dst$$Register,
12670 $src1$$Register,
12671 zr,
12672 Assembler::GT);
12673 %}
12674 ins_pipe(icond_reg);
12675 %}
12676
12677 // This pattern is automatically generated from aarch64_ad.m4
12678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12679 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12680 %{
12681 effect(DEF dst, USE src1, USE cr);
12682 ins_cost(INSN_COST * 2);
12683 format %{ "csinvw $dst, $src1, zr lt\t" %}
12684
12685 ins_encode %{
12686 __ csinvw($dst$$Register,
12687 $src1$$Register,
12688 zr,
12689 Assembler::LT);
12690 %}
12691 ins_pipe(icond_reg);
12692 %}
12693
12694 // This pattern is automatically generated from aarch64_ad.m4
12695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12696 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12697 %{
12698 effect(DEF dst, USE src1, USE cr);
12699 ins_cost(INSN_COST * 2);
12700 format %{ "csinvw $dst, $src1, zr ge\t" %}
12701
12702 ins_encode %{
12703 __ csinvw($dst$$Register,
12704 $src1$$Register,
12705 zr,
12706 Assembler::GE);
12707 %}
12708 ins_pipe(icond_reg);
12709 %}
12710
12711 // This pattern is automatically generated from aarch64_ad.m4
12712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12713 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12714 %{
12715 match(Set dst (MinI src imm));
12716 ins_cost(INSN_COST * 3);
12717 expand %{
12718 rFlagsReg cr;
12719 compI_reg_imm0(cr, src);
12720 cmovI_reg_imm0_lt(dst, src, cr);
12721 %}
12722 %}
12723
12724 // This pattern is automatically generated from aarch64_ad.m4
12725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12726 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12727 %{
12728 match(Set dst (MinI imm src));
12729 ins_cost(INSN_COST * 3);
12730 expand %{
12731 rFlagsReg cr;
12732 compI_reg_imm0(cr, src);
12733 cmovI_reg_imm0_lt(dst, src, cr);
12734 %}
12735 %}
12736
12737 // This pattern is automatically generated from aarch64_ad.m4
12738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12739 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12740 %{
12741 match(Set dst (MinI src imm));
12742 ins_cost(INSN_COST * 3);
12743 expand %{
12744 rFlagsReg cr;
12745 compI_reg_imm0(cr, src);
12746 cmovI_reg_imm1_le(dst, src, cr);
12747 %}
12748 %}
12749
12750 // This pattern is automatically generated from aarch64_ad.m4
12751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12752 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12753 %{
12754 match(Set dst (MinI imm src));
12755 ins_cost(INSN_COST * 3);
12756 expand %{
12757 rFlagsReg cr;
12758 compI_reg_imm0(cr, src);
12759 cmovI_reg_imm1_le(dst, src, cr);
12760 %}
12761 %}
12762
12763 // This pattern is automatically generated from aarch64_ad.m4
12764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12765 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12766 %{
12767 match(Set dst (MinI src imm));
12768 ins_cost(INSN_COST * 3);
12769 expand %{
12770 rFlagsReg cr;
12771 compI_reg_imm0(cr, src);
12772 cmovI_reg_immM1_lt(dst, src, cr);
12773 %}
12774 %}
12775
12776 // This pattern is automatically generated from aarch64_ad.m4
12777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12778 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12779 %{
12780 match(Set dst (MinI imm src));
12781 ins_cost(INSN_COST * 3);
12782 expand %{
12783 rFlagsReg cr;
12784 compI_reg_imm0(cr, src);
12785 cmovI_reg_immM1_lt(dst, src, cr);
12786 %}
12787 %}
12788
12789 // This pattern is automatically generated from aarch64_ad.m4
12790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12791 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12792 %{
12793 match(Set dst (MaxI src imm));
12794 ins_cost(INSN_COST * 3);
12795 expand %{
12796 rFlagsReg cr;
12797 compI_reg_imm0(cr, src);
12798 cmovI_reg_imm0_gt(dst, src, cr);
12799 %}
12800 %}
12801
12802 // This pattern is automatically generated from aarch64_ad.m4
12803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12804 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12805 %{
12806 match(Set dst (MaxI imm src));
12807 ins_cost(INSN_COST * 3);
12808 expand %{
12809 rFlagsReg cr;
12810 compI_reg_imm0(cr, src);
12811 cmovI_reg_imm0_gt(dst, src, cr);
12812 %}
12813 %}
12814
12815 // This pattern is automatically generated from aarch64_ad.m4
12816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12817 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12818 %{
12819 match(Set dst (MaxI src imm));
12820 ins_cost(INSN_COST * 3);
12821 expand %{
12822 rFlagsReg cr;
12823 compI_reg_imm0(cr, src);
12824 cmovI_reg_imm1_gt(dst, src, cr);
12825 %}
12826 %}
12827
12828 // This pattern is automatically generated from aarch64_ad.m4
12829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12830 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12831 %{
12832 match(Set dst (MaxI imm src));
12833 ins_cost(INSN_COST * 3);
12834 expand %{
12835 rFlagsReg cr;
12836 compI_reg_imm0(cr, src);
12837 cmovI_reg_imm1_gt(dst, src, cr);
12838 %}
12839 %}
12840
12841 // This pattern is automatically generated from aarch64_ad.m4
12842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12843 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12844 %{
12845 match(Set dst (MaxI src imm));
12846 ins_cost(INSN_COST * 3);
12847 expand %{
12848 rFlagsReg cr;
12849 compI_reg_imm0(cr, src);
12850 cmovI_reg_immM1_ge(dst, src, cr);
12851 %}
12852 %}
12853
12854 // This pattern is automatically generated from aarch64_ad.m4
12855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12856 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12857 %{
12858 match(Set dst (MaxI imm src));
12859 ins_cost(INSN_COST * 3);
12860 expand %{
12861 rFlagsReg cr;
12862 compI_reg_imm0(cr, src);
12863 cmovI_reg_immM1_ge(dst, src, cr);
12864 %}
12865 %}
12866
12867 // This pattern is automatically generated from aarch64_ad.m4
12868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12869 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12870 %{
12871 match(Set dst (ReverseI src));
12872 ins_cost(INSN_COST);
12873 format %{ "rbitw $dst, $src" %}
12874 ins_encode %{
12875 __ rbitw($dst$$Register, $src$$Register);
12876 %}
12877 ins_pipe(ialu_reg);
12878 %}
12879
12880 // This pattern is automatically generated from aarch64_ad.m4
12881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12882 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12883 %{
12884 match(Set dst (ReverseL src));
12885 ins_cost(INSN_COST);
12886 format %{ "rbit $dst, $src" %}
12887 ins_encode %{
12888 __ rbit($dst$$Register, $src$$Register);
12889 %}
12890 ins_pipe(ialu_reg);
12891 %}
12892
12893
12894 // END This section of the file is automatically generated. Do not edit --------------
12895
12896
12897 // ============================================================================
12898 // Floating Point Arithmetic Instructions
12899
12900 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12901 match(Set dst (AddHF src1 src2));
12902 format %{ "faddh $dst, $src1, $src2" %}
12903 ins_encode %{
12904 __ faddh($dst$$FloatRegister,
12905 $src1$$FloatRegister,
12906 $src2$$FloatRegister);
12907 %}
12908 ins_pipe(fp_dop_reg_reg_s);
12909 %}
12910
12911 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12912 match(Set dst (AddF src1 src2));
12913
12914 ins_cost(INSN_COST * 5);
12915 format %{ "fadds $dst, $src1, $src2" %}
12916
12917 ins_encode %{
12918 __ fadds(as_FloatRegister($dst$$reg),
12919 as_FloatRegister($src1$$reg),
12920 as_FloatRegister($src2$$reg));
12921 %}
12922
12923 ins_pipe(fp_dop_reg_reg_s);
12924 %}
12925
12926 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12927 match(Set dst (AddD src1 src2));
12928
12929 ins_cost(INSN_COST * 5);
12930 format %{ "faddd $dst, $src1, $src2" %}
12931
12932 ins_encode %{
12933 __ faddd(as_FloatRegister($dst$$reg),
12934 as_FloatRegister($src1$$reg),
12935 as_FloatRegister($src2$$reg));
12936 %}
12937
12938 ins_pipe(fp_dop_reg_reg_d);
12939 %}
12940
12941 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12942 match(Set dst (SubHF src1 src2));
12943 format %{ "fsubh $dst, $src1, $src2" %}
12944 ins_encode %{
12945 __ fsubh($dst$$FloatRegister,
12946 $src1$$FloatRegister,
12947 $src2$$FloatRegister);
12948 %}
12949 ins_pipe(fp_dop_reg_reg_s);
12950 %}
12951
12952 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12953 match(Set dst (SubF src1 src2));
12954
12955 ins_cost(INSN_COST * 5);
12956 format %{ "fsubs $dst, $src1, $src2" %}
12957
12958 ins_encode %{
12959 __ fsubs(as_FloatRegister($dst$$reg),
12960 as_FloatRegister($src1$$reg),
12961 as_FloatRegister($src2$$reg));
12962 %}
12963
12964 ins_pipe(fp_dop_reg_reg_s);
12965 %}
12966
12967 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12968 match(Set dst (SubD src1 src2));
12969
12970 ins_cost(INSN_COST * 5);
12971 format %{ "fsubd $dst, $src1, $src2" %}
12972
12973 ins_encode %{
12974 __ fsubd(as_FloatRegister($dst$$reg),
12975 as_FloatRegister($src1$$reg),
12976 as_FloatRegister($src2$$reg));
12977 %}
12978
12979 ins_pipe(fp_dop_reg_reg_d);
12980 %}
12981
12982 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12983 match(Set dst (MulHF src1 src2));
12984 format %{ "fmulh $dst, $src1, $src2" %}
12985 ins_encode %{
12986 __ fmulh($dst$$FloatRegister,
12987 $src1$$FloatRegister,
12988 $src2$$FloatRegister);
12989 %}
12990 ins_pipe(fp_dop_reg_reg_s);
12991 %}
12992
12993 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12994 match(Set dst (MulF src1 src2));
12995
12996 ins_cost(INSN_COST * 6);
12997 format %{ "fmuls $dst, $src1, $src2" %}
12998
12999 ins_encode %{
13000 __ fmuls(as_FloatRegister($dst$$reg),
13001 as_FloatRegister($src1$$reg),
13002 as_FloatRegister($src2$$reg));
13003 %}
13004
13005 ins_pipe(fp_dop_reg_reg_s);
13006 %}
13007
13008 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13009 match(Set dst (MulD src1 src2));
13010
13011 ins_cost(INSN_COST * 6);
13012 format %{ "fmuld $dst, $src1, $src2" %}
13013
13014 ins_encode %{
13015 __ fmuld(as_FloatRegister($dst$$reg),
13016 as_FloatRegister($src1$$reg),
13017 as_FloatRegister($src2$$reg));
13018 %}
13019
13020 ins_pipe(fp_dop_reg_reg_d);
13021 %}
13022
13023 // src1 * src2 + src3 (half-precision float)
13024 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13025 match(Set dst (FmaHF src3 (Binary src1 src2)));
13026 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13027 ins_encode %{
13028 assert(UseFMA, "Needs FMA instructions support.");
13029 __ fmaddh($dst$$FloatRegister,
13030 $src1$$FloatRegister,
13031 $src2$$FloatRegister,
13032 $src3$$FloatRegister);
13033 %}
13034 ins_pipe(pipe_class_default);
13035 %}
13036
13037 // src1 * src2 + src3
13038 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13039 match(Set dst (FmaF src3 (Binary src1 src2)));
13040
13041 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13042
13043 ins_encode %{
13044 assert(UseFMA, "Needs FMA instructions support.");
13045 __ fmadds(as_FloatRegister($dst$$reg),
13046 as_FloatRegister($src1$$reg),
13047 as_FloatRegister($src2$$reg),
13048 as_FloatRegister($src3$$reg));
13049 %}
13050
13051 ins_pipe(pipe_class_default);
13052 %}
13053
13054 // src1 * src2 + src3
13055 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13056 match(Set dst (FmaD src3 (Binary src1 src2)));
13057
13058 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13059
13060 ins_encode %{
13061 assert(UseFMA, "Needs FMA instructions support.");
13062 __ fmaddd(as_FloatRegister($dst$$reg),
13063 as_FloatRegister($src1$$reg),
13064 as_FloatRegister($src2$$reg),
13065 as_FloatRegister($src3$$reg));
13066 %}
13067
13068 ins_pipe(pipe_class_default);
13069 %}
13070
13071 // src1 * (-src2) + src3
13072 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13073 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13074 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13075
13076 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13077
13078 ins_encode %{
13079 assert(UseFMA, "Needs FMA instructions support.");
13080 __ fmsubs(as_FloatRegister($dst$$reg),
13081 as_FloatRegister($src1$$reg),
13082 as_FloatRegister($src2$$reg),
13083 as_FloatRegister($src3$$reg));
13084 %}
13085
13086 ins_pipe(pipe_class_default);
13087 %}
13088
13089 // src1 * (-src2) + src3
13090 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13091 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13092 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13093
13094 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13095
13096 ins_encode %{
13097 assert(UseFMA, "Needs FMA instructions support.");
13098 __ fmsubd(as_FloatRegister($dst$$reg),
13099 as_FloatRegister($src1$$reg),
13100 as_FloatRegister($src2$$reg),
13101 as_FloatRegister($src3$$reg));
13102 %}
13103
13104 ins_pipe(pipe_class_default);
13105 %}
13106
13107 // src1 * (-src2) - src3
13108 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13109 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13110 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13111
13112 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13113
13114 ins_encode %{
13115 assert(UseFMA, "Needs FMA instructions support.");
13116 __ fnmadds(as_FloatRegister($dst$$reg),
13117 as_FloatRegister($src1$$reg),
13118 as_FloatRegister($src2$$reg),
13119 as_FloatRegister($src3$$reg));
13120 %}
13121
13122 ins_pipe(pipe_class_default);
13123 %}
13124
13125 // src1 * (-src2) - src3
13126 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13127 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13128 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13129
13130 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13131
13132 ins_encode %{
13133 assert(UseFMA, "Needs FMA instructions support.");
13134 __ fnmaddd(as_FloatRegister($dst$$reg),
13135 as_FloatRegister($src1$$reg),
13136 as_FloatRegister($src2$$reg),
13137 as_FloatRegister($src3$$reg));
13138 %}
13139
13140 ins_pipe(pipe_class_default);
13141 %}
13142
13143 // src1 * src2 - src3
13144 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13145 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13146
13147 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13148
13149 ins_encode %{
13150 assert(UseFMA, "Needs FMA instructions support.");
13151 __ fnmsubs(as_FloatRegister($dst$$reg),
13152 as_FloatRegister($src1$$reg),
13153 as_FloatRegister($src2$$reg),
13154 as_FloatRegister($src3$$reg));
13155 %}
13156
13157 ins_pipe(pipe_class_default);
13158 %}
13159
13160 // src1 * src2 - src3
13161 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13162 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13163
13164 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13165
13166 ins_encode %{
13167 assert(UseFMA, "Needs FMA instructions support.");
13168 // n.b. insn name should be fnmsubd
13169 __ fnmsub(as_FloatRegister($dst$$reg),
13170 as_FloatRegister($src1$$reg),
13171 as_FloatRegister($src2$$reg),
13172 as_FloatRegister($src3$$reg));
13173 %}
13174
13175 ins_pipe(pipe_class_default);
13176 %}
13177
13178 // Math.max(HH)H (half-precision float)
13179 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13180 match(Set dst (MaxHF src1 src2));
13181 format %{ "fmaxh $dst, $src1, $src2" %}
13182 ins_encode %{
13183 __ fmaxh($dst$$FloatRegister,
13184 $src1$$FloatRegister,
13185 $src2$$FloatRegister);
13186 %}
13187 ins_pipe(fp_dop_reg_reg_s);
13188 %}
13189
13190 // Math.min(HH)H (half-precision float)
13191 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13192 match(Set dst (MinHF src1 src2));
13193 format %{ "fminh $dst, $src1, $src2" %}
13194 ins_encode %{
13195 __ fminh($dst$$FloatRegister,
13196 $src1$$FloatRegister,
13197 $src2$$FloatRegister);
13198 %}
13199 ins_pipe(fp_dop_reg_reg_s);
13200 %}
13201
13202 // Math.max(FF)F
13203 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13204 match(Set dst (MaxF src1 src2));
13205
13206 format %{ "fmaxs $dst, $src1, $src2" %}
13207 ins_encode %{
13208 __ fmaxs(as_FloatRegister($dst$$reg),
13209 as_FloatRegister($src1$$reg),
13210 as_FloatRegister($src2$$reg));
13211 %}
13212
13213 ins_pipe(fp_dop_reg_reg_s);
13214 %}
13215
13216 // Math.min(FF)F
13217 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13218 match(Set dst (MinF src1 src2));
13219
13220 format %{ "fmins $dst, $src1, $src2" %}
13221 ins_encode %{
13222 __ fmins(as_FloatRegister($dst$$reg),
13223 as_FloatRegister($src1$$reg),
13224 as_FloatRegister($src2$$reg));
13225 %}
13226
13227 ins_pipe(fp_dop_reg_reg_s);
13228 %}
13229
13230 // Math.max(DD)D
13231 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13232 match(Set dst (MaxD src1 src2));
13233
13234 format %{ "fmaxd $dst, $src1, $src2" %}
13235 ins_encode %{
13236 __ fmaxd(as_FloatRegister($dst$$reg),
13237 as_FloatRegister($src1$$reg),
13238 as_FloatRegister($src2$$reg));
13239 %}
13240
13241 ins_pipe(fp_dop_reg_reg_d);
13242 %}
13243
13244 // Math.min(DD)D
13245 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13246 match(Set dst (MinD src1 src2));
13247
13248 format %{ "fmind $dst, $src1, $src2" %}
13249 ins_encode %{
13250 __ fmind(as_FloatRegister($dst$$reg),
13251 as_FloatRegister($src1$$reg),
13252 as_FloatRegister($src2$$reg));
13253 %}
13254
13255 ins_pipe(fp_dop_reg_reg_d);
13256 %}
13257
13258 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13259 match(Set dst (DivHF src1 src2));
13260 format %{ "fdivh $dst, $src1, $src2" %}
13261 ins_encode %{
13262 __ fdivh($dst$$FloatRegister,
13263 $src1$$FloatRegister,
13264 $src2$$FloatRegister);
13265 %}
13266 ins_pipe(fp_div_s);
13267 %}
13268
13269 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13270 match(Set dst (DivF src1 src2));
13271
13272 ins_cost(INSN_COST * 18);
13273 format %{ "fdivs $dst, $src1, $src2" %}
13274
13275 ins_encode %{
13276 __ fdivs(as_FloatRegister($dst$$reg),
13277 as_FloatRegister($src1$$reg),
13278 as_FloatRegister($src2$$reg));
13279 %}
13280
13281 ins_pipe(fp_div_s);
13282 %}
13283
13284 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13285 match(Set dst (DivD src1 src2));
13286
13287 ins_cost(INSN_COST * 32);
13288 format %{ "fdivd $dst, $src1, $src2" %}
13289
13290 ins_encode %{
13291 __ fdivd(as_FloatRegister($dst$$reg),
13292 as_FloatRegister($src1$$reg),
13293 as_FloatRegister($src2$$reg));
13294 %}
13295
13296 ins_pipe(fp_div_d);
13297 %}
13298
13299 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13300 match(Set dst (NegF src));
13301
13302 ins_cost(INSN_COST * 3);
13303 format %{ "fneg $dst, $src" %}
13304
13305 ins_encode %{
13306 __ fnegs(as_FloatRegister($dst$$reg),
13307 as_FloatRegister($src$$reg));
13308 %}
13309
13310 ins_pipe(fp_uop_s);
13311 %}
13312
13313 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13314 match(Set dst (NegD src));
13315
13316 ins_cost(INSN_COST * 3);
13317 format %{ "fnegd $dst, $src" %}
13318
13319 ins_encode %{
13320 __ fnegd(as_FloatRegister($dst$$reg),
13321 as_FloatRegister($src$$reg));
13322 %}
13323
13324 ins_pipe(fp_uop_d);
13325 %}
13326
13327 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13328 %{
13329 match(Set dst (AbsI src));
13330
13331 effect(KILL cr);
13332 ins_cost(INSN_COST * 2);
13333 format %{ "cmpw $src, zr\n\t"
13334 "cnegw $dst, $src, Assembler::LT\t# int abs"
13335 %}
13336
13337 ins_encode %{
13338 __ cmpw(as_Register($src$$reg), zr);
13339 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13340 %}
13341 ins_pipe(pipe_class_default);
13342 %}
13343
13344 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13345 %{
13346 match(Set dst (AbsL src));
13347
13348 effect(KILL cr);
13349 ins_cost(INSN_COST * 2);
13350 format %{ "cmp $src, zr\n\t"
13351 "cneg $dst, $src, Assembler::LT\t# long abs"
13352 %}
13353
13354 ins_encode %{
13355 __ cmp(as_Register($src$$reg), zr);
13356 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13357 %}
13358 ins_pipe(pipe_class_default);
13359 %}
13360
13361 instruct absF_reg(vRegF dst, vRegF src) %{
13362 match(Set dst (AbsF src));
13363
13364 ins_cost(INSN_COST * 3);
13365 format %{ "fabss $dst, $src" %}
13366 ins_encode %{
13367 __ fabss(as_FloatRegister($dst$$reg),
13368 as_FloatRegister($src$$reg));
13369 %}
13370
13371 ins_pipe(fp_uop_s);
13372 %}
13373
13374 instruct absD_reg(vRegD dst, vRegD src) %{
13375 match(Set dst (AbsD src));
13376
13377 ins_cost(INSN_COST * 3);
13378 format %{ "fabsd $dst, $src" %}
13379 ins_encode %{
13380 __ fabsd(as_FloatRegister($dst$$reg),
13381 as_FloatRegister($src$$reg));
13382 %}
13383
13384 ins_pipe(fp_uop_d);
13385 %}
13386
13387 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13388 match(Set dst (AbsF (SubF src1 src2)));
13389
13390 ins_cost(INSN_COST * 3);
13391 format %{ "fabds $dst, $src1, $src2" %}
13392 ins_encode %{
13393 __ fabds(as_FloatRegister($dst$$reg),
13394 as_FloatRegister($src1$$reg),
13395 as_FloatRegister($src2$$reg));
13396 %}
13397
13398 ins_pipe(fp_uop_s);
13399 %}
13400
13401 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13402 match(Set dst (AbsD (SubD src1 src2)));
13403
13404 ins_cost(INSN_COST * 3);
13405 format %{ "fabdd $dst, $src1, $src2" %}
13406 ins_encode %{
13407 __ fabdd(as_FloatRegister($dst$$reg),
13408 as_FloatRegister($src1$$reg),
13409 as_FloatRegister($src2$$reg));
13410 %}
13411
13412 ins_pipe(fp_uop_d);
13413 %}
13414
13415 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13416 match(Set dst (SqrtD src));
13417
13418 ins_cost(INSN_COST * 50);
13419 format %{ "fsqrtd $dst, $src" %}
13420 ins_encode %{
13421 __ fsqrtd(as_FloatRegister($dst$$reg),
13422 as_FloatRegister($src$$reg));
13423 %}
13424
13425 ins_pipe(fp_div_s);
13426 %}
13427
13428 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13429 match(Set dst (SqrtF src));
13430
13431 ins_cost(INSN_COST * 50);
13432 format %{ "fsqrts $dst, $src" %}
13433 ins_encode %{
13434 __ fsqrts(as_FloatRegister($dst$$reg),
13435 as_FloatRegister($src$$reg));
13436 %}
13437
13438 ins_pipe(fp_div_d);
13439 %}
13440
13441 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13442 match(Set dst (SqrtHF src));
13443 format %{ "fsqrth $dst, $src" %}
13444 ins_encode %{
13445 __ fsqrth($dst$$FloatRegister,
13446 $src$$FloatRegister);
13447 %}
13448 ins_pipe(fp_div_s);
13449 %}
13450
13451 // Math.rint, floor, ceil
13452 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13453 match(Set dst (RoundDoubleMode src rmode));
13454 format %{ "frint $dst, $src, $rmode" %}
13455 ins_encode %{
13456 switch ($rmode$$constant) {
13457 case RoundDoubleModeNode::rmode_rint:
13458 __ frintnd(as_FloatRegister($dst$$reg),
13459 as_FloatRegister($src$$reg));
13460 break;
13461 case RoundDoubleModeNode::rmode_floor:
13462 __ frintmd(as_FloatRegister($dst$$reg),
13463 as_FloatRegister($src$$reg));
13464 break;
13465 case RoundDoubleModeNode::rmode_ceil:
13466 __ frintpd(as_FloatRegister($dst$$reg),
13467 as_FloatRegister($src$$reg));
13468 break;
13469 }
13470 %}
13471 ins_pipe(fp_uop_d);
13472 %}
13473
13474 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13475 match(Set dst (CopySignD src1 (Binary src2 zero)));
13476 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13477 format %{ "CopySignD $dst $src1 $src2" %}
13478 ins_encode %{
13479 FloatRegister dst = as_FloatRegister($dst$$reg),
13480 src1 = as_FloatRegister($src1$$reg),
13481 src2 = as_FloatRegister($src2$$reg),
13482 zero = as_FloatRegister($zero$$reg);
13483 __ fnegd(dst, zero);
13484 __ bsl(dst, __ T8B, src2, src1);
13485 %}
13486 ins_pipe(fp_uop_d);
13487 %}
13488
13489 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13490 match(Set dst (CopySignF src1 src2));
13491 effect(TEMP_DEF dst, USE src1, USE src2);
13492 format %{ "CopySignF $dst $src1 $src2" %}
13493 ins_encode %{
13494 FloatRegister dst = as_FloatRegister($dst$$reg),
13495 src1 = as_FloatRegister($src1$$reg),
13496 src2 = as_FloatRegister($src2$$reg);
13497 __ movi(dst, __ T2S, 0x80, 24);
13498 __ bsl(dst, __ T8B, src2, src1);
13499 %}
13500 ins_pipe(fp_uop_d);
13501 %}
13502
13503 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13504 match(Set dst (SignumD src (Binary zero one)));
13505 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13506 format %{ "signumD $dst, $src" %}
13507 ins_encode %{
13508 FloatRegister src = as_FloatRegister($src$$reg),
13509 dst = as_FloatRegister($dst$$reg),
13510 zero = as_FloatRegister($zero$$reg),
13511 one = as_FloatRegister($one$$reg);
13512 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13513 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13514 // Bit selection instruction gets bit from "one" for each enabled bit in
13515 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13516 // NaN the whole "src" will be copied because "dst" is zero. For all other
13517 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13518 // from "src", and all other bits are copied from 1.0.
13519 __ bsl(dst, __ T8B, one, src);
13520 %}
13521 ins_pipe(fp_uop_d);
13522 %}
13523
13524 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13525 match(Set dst (SignumF src (Binary zero one)));
13526 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13527 format %{ "signumF $dst, $src" %}
13528 ins_encode %{
13529 FloatRegister src = as_FloatRegister($src$$reg),
13530 dst = as_FloatRegister($dst$$reg),
13531 zero = as_FloatRegister($zero$$reg),
13532 one = as_FloatRegister($one$$reg);
13533 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13534 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13535 // Bit selection instruction gets bit from "one" for each enabled bit in
13536 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13537 // NaN the whole "src" will be copied because "dst" is zero. For all other
13538 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13539 // from "src", and all other bits are copied from 1.0.
13540 __ bsl(dst, __ T8B, one, src);
13541 %}
13542 ins_pipe(fp_uop_d);
13543 %}
13544
13545 instruct onspinwait() %{
13546 match(OnSpinWait);
13547 ins_cost(INSN_COST);
13548
13549 format %{ "onspinwait" %}
13550
13551 ins_encode %{
13552 __ spin_wait();
13553 %}
13554 ins_pipe(pipe_class_empty);
13555 %}
13556
13557 // ============================================================================
13558 // Logical Instructions
13559
13560 // Integer Logical Instructions
13561
13562 // And Instructions
13563
13564
13565 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13566 match(Set dst (AndI src1 src2));
13567
13568 format %{ "andw $dst, $src1, $src2\t# int" %}
13569
13570 ins_cost(INSN_COST);
13571 ins_encode %{
13572 __ andw(as_Register($dst$$reg),
13573 as_Register($src1$$reg),
13574 as_Register($src2$$reg));
13575 %}
13576
13577 ins_pipe(ialu_reg_reg);
13578 %}
13579
13580 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13581 match(Set dst (AndI src1 src2));
13582
13583 format %{ "andsw $dst, $src1, $src2\t# int" %}
13584
13585 ins_cost(INSN_COST);
13586 ins_encode %{
13587 __ andw(as_Register($dst$$reg),
13588 as_Register($src1$$reg),
13589 (uint64_t)($src2$$constant));
13590 %}
13591
13592 ins_pipe(ialu_reg_imm);
13593 %}
13594
13595 // Or Instructions
13596
13597 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13598 match(Set dst (OrI src1 src2));
13599
13600 format %{ "orrw $dst, $src1, $src2\t# int" %}
13601
13602 ins_cost(INSN_COST);
13603 ins_encode %{
13604 __ orrw(as_Register($dst$$reg),
13605 as_Register($src1$$reg),
13606 as_Register($src2$$reg));
13607 %}
13608
13609 ins_pipe(ialu_reg_reg);
13610 %}
13611
13612 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13613 match(Set dst (OrI src1 src2));
13614
13615 format %{ "orrw $dst, $src1, $src2\t# int" %}
13616
13617 ins_cost(INSN_COST);
13618 ins_encode %{
13619 __ orrw(as_Register($dst$$reg),
13620 as_Register($src1$$reg),
13621 (uint64_t)($src2$$constant));
13622 %}
13623
13624 ins_pipe(ialu_reg_imm);
13625 %}
13626
13627 // Xor Instructions
13628
13629 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13630 match(Set dst (XorI src1 src2));
13631
13632 format %{ "eorw $dst, $src1, $src2\t# int" %}
13633
13634 ins_cost(INSN_COST);
13635 ins_encode %{
13636 __ eorw(as_Register($dst$$reg),
13637 as_Register($src1$$reg),
13638 as_Register($src2$$reg));
13639 %}
13640
13641 ins_pipe(ialu_reg_reg);
13642 %}
13643
13644 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13645 match(Set dst (XorI src1 src2));
13646
13647 format %{ "eorw $dst, $src1, $src2\t# int" %}
13648
13649 ins_cost(INSN_COST);
13650 ins_encode %{
13651 __ eorw(as_Register($dst$$reg),
13652 as_Register($src1$$reg),
13653 (uint64_t)($src2$$constant));
13654 %}
13655
13656 ins_pipe(ialu_reg_imm);
13657 %}
13658
13659 // Long Logical Instructions
13660 // TODO
13661
13662 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13663 match(Set dst (AndL src1 src2));
13664
13665 format %{ "and $dst, $src1, $src2\t# int" %}
13666
13667 ins_cost(INSN_COST);
13668 ins_encode %{
13669 __ andr(as_Register($dst$$reg),
13670 as_Register($src1$$reg),
13671 as_Register($src2$$reg));
13672 %}
13673
13674 ins_pipe(ialu_reg_reg);
13675 %}
13676
13677 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13678 match(Set dst (AndL src1 src2));
13679
13680 format %{ "and $dst, $src1, $src2\t# int" %}
13681
13682 ins_cost(INSN_COST);
13683 ins_encode %{
13684 __ andr(as_Register($dst$$reg),
13685 as_Register($src1$$reg),
13686 (uint64_t)($src2$$constant));
13687 %}
13688
13689 ins_pipe(ialu_reg_imm);
13690 %}
13691
13692 // Or Instructions
13693
13694 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13695 match(Set dst (OrL src1 src2));
13696
13697 format %{ "orr $dst, $src1, $src2\t# int" %}
13698
13699 ins_cost(INSN_COST);
13700 ins_encode %{
13701 __ orr(as_Register($dst$$reg),
13702 as_Register($src1$$reg),
13703 as_Register($src2$$reg));
13704 %}
13705
13706 ins_pipe(ialu_reg_reg);
13707 %}
13708
13709 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13710 match(Set dst (OrL src1 src2));
13711
13712 format %{ "orr $dst, $src1, $src2\t# int" %}
13713
13714 ins_cost(INSN_COST);
13715 ins_encode %{
13716 __ orr(as_Register($dst$$reg),
13717 as_Register($src1$$reg),
13718 (uint64_t)($src2$$constant));
13719 %}
13720
13721 ins_pipe(ialu_reg_imm);
13722 %}
13723
13724 // Xor Instructions
13725
13726 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13727 match(Set dst (XorL src1 src2));
13728
13729 format %{ "eor $dst, $src1, $src2\t# int" %}
13730
13731 ins_cost(INSN_COST);
13732 ins_encode %{
13733 __ eor(as_Register($dst$$reg),
13734 as_Register($src1$$reg),
13735 as_Register($src2$$reg));
13736 %}
13737
13738 ins_pipe(ialu_reg_reg);
13739 %}
13740
13741 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13742 match(Set dst (XorL src1 src2));
13743
13744 ins_cost(INSN_COST);
13745 format %{ "eor $dst, $src1, $src2\t# int" %}
13746
13747 ins_encode %{
13748 __ eor(as_Register($dst$$reg),
13749 as_Register($src1$$reg),
13750 (uint64_t)($src2$$constant));
13751 %}
13752
13753 ins_pipe(ialu_reg_imm);
13754 %}
13755
13756 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13757 %{
13758 match(Set dst (ConvI2L src));
13759
13760 ins_cost(INSN_COST);
13761 format %{ "sxtw $dst, $src\t# i2l" %}
13762 ins_encode %{
13763 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13764 %}
13765 ins_pipe(ialu_reg_shift);
13766 %}
13767
13768 // this pattern occurs in bigmath arithmetic
13769 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13770 %{
13771 match(Set dst (AndL (ConvI2L src) mask));
13772
13773 ins_cost(INSN_COST);
13774 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13775 ins_encode %{
13776 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13777 %}
13778
13779 ins_pipe(ialu_reg_shift);
13780 %}
13781
13782 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13783 match(Set dst (ConvL2I src));
13784
13785 ins_cost(INSN_COST);
13786 format %{ "movw $dst, $src \t// l2i" %}
13787
13788 ins_encode %{
13789 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13790 %}
13791
13792 ins_pipe(ialu_reg);
13793 %}
13794
13795 instruct convD2F_reg(vRegF dst, vRegD src) %{
13796 match(Set dst (ConvD2F src));
13797
13798 ins_cost(INSN_COST * 5);
13799 format %{ "fcvtd $dst, $src \t// d2f" %}
13800
13801 ins_encode %{
13802 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13803 %}
13804
13805 ins_pipe(fp_d2f);
13806 %}
13807
13808 instruct convF2D_reg(vRegD dst, vRegF src) %{
13809 match(Set dst (ConvF2D src));
13810
13811 ins_cost(INSN_COST * 5);
13812 format %{ "fcvts $dst, $src \t// f2d" %}
13813
13814 ins_encode %{
13815 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13816 %}
13817
13818 ins_pipe(fp_f2d);
13819 %}
13820
13821 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13822 match(Set dst (ConvF2I src));
13823
13824 ins_cost(INSN_COST * 5);
13825 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13826
13827 ins_encode %{
13828 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13829 %}
13830
13831 ins_pipe(fp_f2i);
13832 %}
13833
13834 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13835 match(Set dst (ConvF2L src));
13836
13837 ins_cost(INSN_COST * 5);
13838 format %{ "fcvtzs $dst, $src \t// f2l" %}
13839
13840 ins_encode %{
13841 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13842 %}
13843
13844 ins_pipe(fp_f2l);
13845 %}
13846
13847 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13848 match(Set dst (ConvF2HF src));
13849 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13850 "smov $dst, $tmp\t# move result from $tmp to $dst"
13851 %}
13852 effect(TEMP tmp);
13853 ins_encode %{
13854 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13855 %}
13856 ins_pipe(pipe_slow);
13857 %}
13858
13859 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13860 match(Set dst (ConvHF2F src));
13861 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13862 "fcvt $dst, $tmp\t# convert half to single precision"
13863 %}
13864 effect(TEMP tmp);
13865 ins_encode %{
13866 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13867 %}
13868 ins_pipe(pipe_slow);
13869 %}
13870
13871 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13872 match(Set dst (ConvI2F src));
13873
13874 ins_cost(INSN_COST * 5);
13875 format %{ "scvtfws $dst, $src \t// i2f" %}
13876
13877 ins_encode %{
13878 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13879 %}
13880
13881 ins_pipe(fp_i2f);
13882 %}
13883
13884 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13885 match(Set dst (ConvL2F src));
13886
13887 ins_cost(INSN_COST * 5);
13888 format %{ "scvtfs $dst, $src \t// l2f" %}
13889
13890 ins_encode %{
13891 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13892 %}
13893
13894 ins_pipe(fp_l2f);
13895 %}
13896
13897 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13898 match(Set dst (ConvD2I src));
13899
13900 ins_cost(INSN_COST * 5);
13901 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13902
13903 ins_encode %{
13904 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13905 %}
13906
13907 ins_pipe(fp_d2i);
13908 %}
13909
13910 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13911 match(Set dst (ConvD2L src));
13912
13913 ins_cost(INSN_COST * 5);
13914 format %{ "fcvtzd $dst, $src \t// d2l" %}
13915
13916 ins_encode %{
13917 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13918 %}
13919
13920 ins_pipe(fp_d2l);
13921 %}
13922
13923 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13924 match(Set dst (ConvI2D src));
13925
13926 ins_cost(INSN_COST * 5);
13927 format %{ "scvtfwd $dst, $src \t// i2d" %}
13928
13929 ins_encode %{
13930 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13931 %}
13932
13933 ins_pipe(fp_i2d);
13934 %}
13935
13936 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13937 match(Set dst (ConvL2D src));
13938
13939 ins_cost(INSN_COST * 5);
13940 format %{ "scvtfd $dst, $src \t// l2d" %}
13941
13942 ins_encode %{
13943 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13944 %}
13945
13946 ins_pipe(fp_l2d);
13947 %}
13948
13949 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13950 %{
13951 match(Set dst (RoundD src));
13952 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13953 format %{ "java_round_double $dst,$src"%}
13954 ins_encode %{
13955 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13956 as_FloatRegister($ftmp$$reg));
13957 %}
13958 ins_pipe(pipe_slow);
13959 %}
13960
13961 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13962 %{
13963 match(Set dst (RoundF src));
13964 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13965 format %{ "java_round_float $dst,$src"%}
13966 ins_encode %{
13967 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13968 as_FloatRegister($ftmp$$reg));
13969 %}
13970 ins_pipe(pipe_slow);
13971 %}
13972
13973 // stack <-> reg and reg <-> reg shuffles with no conversion
13974
13975 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13976
13977 match(Set dst (MoveF2I src));
13978
13979 effect(DEF dst, USE src);
13980
13981 ins_cost(4 * INSN_COST);
13982
13983 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13984
13985 ins_encode %{
13986 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13987 %}
13988
13989 ins_pipe(iload_reg_reg);
13990
13991 %}
13992
13993 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13994
13995 match(Set dst (MoveI2F src));
13996
13997 effect(DEF dst, USE src);
13998
13999 ins_cost(4 * INSN_COST);
14000
14001 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14002
14003 ins_encode %{
14004 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14005 %}
14006
14007 ins_pipe(pipe_class_memory);
14008
14009 %}
14010
14011 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14012
14013 match(Set dst (MoveD2L src));
14014
14015 effect(DEF dst, USE src);
14016
14017 ins_cost(4 * INSN_COST);
14018
14019 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14020
14021 ins_encode %{
14022 __ ldr($dst$$Register, Address(sp, $src$$disp));
14023 %}
14024
14025 ins_pipe(iload_reg_reg);
14026
14027 %}
14028
14029 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14030
14031 match(Set dst (MoveL2D src));
14032
14033 effect(DEF dst, USE src);
14034
14035 ins_cost(4 * INSN_COST);
14036
14037 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14038
14039 ins_encode %{
14040 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14041 %}
14042
14043 ins_pipe(pipe_class_memory);
14044
14045 %}
14046
14047 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14048
14049 match(Set dst (MoveF2I src));
14050
14051 effect(DEF dst, USE src);
14052
14053 ins_cost(INSN_COST);
14054
14055 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14056
14057 ins_encode %{
14058 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14059 %}
14060
14061 ins_pipe(pipe_class_memory);
14062
14063 %}
14064
14065 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14066
14067 match(Set dst (MoveI2F src));
14068
14069 effect(DEF dst, USE src);
14070
14071 ins_cost(INSN_COST);
14072
14073 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14074
14075 ins_encode %{
14076 __ strw($src$$Register, Address(sp, $dst$$disp));
14077 %}
14078
14079 ins_pipe(istore_reg_reg);
14080
14081 %}
14082
14083 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14084
14085 match(Set dst (MoveD2L src));
14086
14087 effect(DEF dst, USE src);
14088
14089 ins_cost(INSN_COST);
14090
14091 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14092
14093 ins_encode %{
14094 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14095 %}
14096
14097 ins_pipe(pipe_class_memory);
14098
14099 %}
14100
14101 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14102
14103 match(Set dst (MoveL2D src));
14104
14105 effect(DEF dst, USE src);
14106
14107 ins_cost(INSN_COST);
14108
14109 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14110
14111 ins_encode %{
14112 __ str($src$$Register, Address(sp, $dst$$disp));
14113 %}
14114
14115 ins_pipe(istore_reg_reg);
14116
14117 %}
14118
14119 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14120
14121 match(Set dst (MoveF2I src));
14122
14123 effect(DEF dst, USE src);
14124
14125 ins_cost(INSN_COST);
14126
14127 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14128
14129 ins_encode %{
14130 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14131 %}
14132
14133 ins_pipe(fp_f2i);
14134
14135 %}
14136
14137 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14138
14139 match(Set dst (MoveI2F src));
14140
14141 effect(DEF dst, USE src);
14142
14143 ins_cost(INSN_COST);
14144
14145 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14146
14147 ins_encode %{
14148 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14149 %}
14150
14151 ins_pipe(fp_i2f);
14152
14153 %}
14154
14155 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14156
14157 match(Set dst (MoveD2L src));
14158
14159 effect(DEF dst, USE src);
14160
14161 ins_cost(INSN_COST);
14162
14163 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14164
14165 ins_encode %{
14166 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14167 %}
14168
14169 ins_pipe(fp_d2l);
14170
14171 %}
14172
14173 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14174
14175 match(Set dst (MoveL2D src));
14176
14177 effect(DEF dst, USE src);
14178
14179 ins_cost(INSN_COST);
14180
14181 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14182
14183 ins_encode %{
14184 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14185 %}
14186
14187 ins_pipe(fp_l2d);
14188
14189 %}
14190
14191 // ============================================================================
14192 // clearing of an array
14193
14194 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14195 %{
14196 match(Set dummy (ClearArray (Binary cnt base) zero));
14197 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14198
14199 ins_cost(4 * INSN_COST);
14200 format %{ "ClearArray $cnt, $base" %}
14201
14202 ins_encode %{
14203 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14204 if (tpc == nullptr) {
14205 ciEnv::current()->record_failure("CodeCache is full");
14206 return;
14207 }
14208 %}
14209
14210 ins_pipe(pipe_class_memory);
14211 %}
14212
14213 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14214 %{
14215 predicate(((ClearArrayNode*)n)->word_copy_only());
14216 match(Set dummy (ClearArray (Binary cnt base) val));
14217 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14218
14219 ins_cost(4 * INSN_COST);
14220 format %{ "ClearArray $cnt, $base, $val" %}
14221
14222 ins_encode %{
14223 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14224 %}
14225
14226 ins_pipe(pipe_class_memory);
14227 %}
14228
14229 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, immL0 zero, Universe dummy, rFlagsReg cr)
14230 %{
14231 predicate((uint64_t)n->in(2)->in(1)->get_long()
14232 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14233 && !((ClearArrayNode*)n)->word_copy_only());
14234 match(Set dummy (ClearArray (Binary cnt base) zero));
14235 effect(TEMP temp, USE_KILL base, KILL cr);
14236
14237 ins_cost(4 * INSN_COST);
14238 format %{ "ClearArray $cnt, $base" %}
14239
14240 ins_encode %{
14241 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14242 if (tpc == nullptr) {
14243 ciEnv::current()->record_failure("CodeCache is full");
14244 return;
14245 }
14246 %}
14247
14248 ins_pipe(pipe_class_memory);
14249 %}
14250
14251 // ============================================================================
14252 // Overflow Math Instructions
14253
14254 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14255 %{
14256 match(Set cr (OverflowAddI op1 op2));
14257
14258 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14259 ins_cost(INSN_COST);
14260 ins_encode %{
14261 __ cmnw($op1$$Register, $op2$$Register);
14262 %}
14263
14264 ins_pipe(icmp_reg_reg);
14265 %}
14266
14267 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14268 %{
14269 match(Set cr (OverflowAddI op1 op2));
14270
14271 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14272 ins_cost(INSN_COST);
14273 ins_encode %{
14274 __ cmnw($op1$$Register, $op2$$constant);
14275 %}
14276
14277 ins_pipe(icmp_reg_imm);
14278 %}
14279
14280 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14281 %{
14282 match(Set cr (OverflowAddL op1 op2));
14283
14284 format %{ "cmn $op1, $op2\t# overflow check long" %}
14285 ins_cost(INSN_COST);
14286 ins_encode %{
14287 __ cmn($op1$$Register, $op2$$Register);
14288 %}
14289
14290 ins_pipe(icmp_reg_reg);
14291 %}
14292
14293 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14294 %{
14295 match(Set cr (OverflowAddL op1 op2));
14296
14297 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14298 ins_cost(INSN_COST);
14299 ins_encode %{
14300 __ adds(zr, $op1$$Register, $op2$$constant);
14301 %}
14302
14303 ins_pipe(icmp_reg_imm);
14304 %}
14305
14306 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14307 %{
14308 match(Set cr (OverflowSubI op1 op2));
14309
14310 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14311 ins_cost(INSN_COST);
14312 ins_encode %{
14313 __ cmpw($op1$$Register, $op2$$Register);
14314 %}
14315
14316 ins_pipe(icmp_reg_reg);
14317 %}
14318
14319 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14320 %{
14321 match(Set cr (OverflowSubI op1 op2));
14322
14323 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14324 ins_cost(INSN_COST);
14325 ins_encode %{
14326 __ cmpw($op1$$Register, $op2$$constant);
14327 %}
14328
14329 ins_pipe(icmp_reg_imm);
14330 %}
14331
14332 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14333 %{
14334 match(Set cr (OverflowSubL op1 op2));
14335
14336 format %{ "cmp $op1, $op2\t# overflow check long" %}
14337 ins_cost(INSN_COST);
14338 ins_encode %{
14339 __ cmp($op1$$Register, $op2$$Register);
14340 %}
14341
14342 ins_pipe(icmp_reg_reg);
14343 %}
14344
14345 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14346 %{
14347 match(Set cr (OverflowSubL op1 op2));
14348
14349 format %{ "cmp $op1, $op2\t# overflow check long" %}
14350 ins_cost(INSN_COST);
14351 ins_encode %{
14352 __ subs(zr, $op1$$Register, $op2$$constant);
14353 %}
14354
14355 ins_pipe(icmp_reg_imm);
14356 %}
14357
14358 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14359 %{
14360 match(Set cr (OverflowSubI zero op1));
14361
14362 format %{ "cmpw zr, $op1\t# overflow check int" %}
14363 ins_cost(INSN_COST);
14364 ins_encode %{
14365 __ cmpw(zr, $op1$$Register);
14366 %}
14367
14368 ins_pipe(icmp_reg_imm);
14369 %}
14370
14371 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14372 %{
14373 match(Set cr (OverflowSubL zero op1));
14374
14375 format %{ "cmp zr, $op1\t# overflow check long" %}
14376 ins_cost(INSN_COST);
14377 ins_encode %{
14378 __ cmp(zr, $op1$$Register);
14379 %}
14380
14381 ins_pipe(icmp_reg_imm);
14382 %}
14383
14384 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14385 %{
14386 match(Set cr (OverflowMulI op1 op2));
14387
14388 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14389 "cmp rscratch1, rscratch1, sxtw\n\t"
14390 "movw rscratch1, #0x80000000\n\t"
14391 "cselw rscratch1, rscratch1, zr, NE\n\t"
14392 "cmpw rscratch1, #1" %}
14393 ins_cost(5 * INSN_COST);
14394 ins_encode %{
14395 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14396 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14397 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14398 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14399 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14400 %}
14401
14402 ins_pipe(pipe_slow);
14403 %}
14404
14405 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14406 %{
14407 match(If cmp (OverflowMulI op1 op2));
14408 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14409 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14410 effect(USE labl, KILL cr);
14411
14412 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14413 "cmp rscratch1, rscratch1, sxtw\n\t"
14414 "b$cmp $labl" %}
14415 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14416 ins_encode %{
14417 Label* L = $labl$$label;
14418 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14419 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14420 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14421 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14422 %}
14423
14424 ins_pipe(pipe_serial);
14425 %}
14426
14427 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14428 %{
14429 match(Set cr (OverflowMulL op1 op2));
14430
14431 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14432 "smulh rscratch2, $op1, $op2\n\t"
14433 "cmp rscratch2, rscratch1, ASR #63\n\t"
14434 "movw rscratch1, #0x80000000\n\t"
14435 "cselw rscratch1, rscratch1, zr, NE\n\t"
14436 "cmpw rscratch1, #1" %}
14437 ins_cost(6 * INSN_COST);
14438 ins_encode %{
14439 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14440 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14441 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14442 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14443 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14444 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14445 %}
14446
14447 ins_pipe(pipe_slow);
14448 %}
14449
14450 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14451 %{
14452 match(If cmp (OverflowMulL op1 op2));
14453 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14454 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14455 effect(USE labl, KILL cr);
14456
14457 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14458 "smulh rscratch2, $op1, $op2\n\t"
14459 "cmp rscratch2, rscratch1, ASR #63\n\t"
14460 "b$cmp $labl" %}
14461 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14462 ins_encode %{
14463 Label* L = $labl$$label;
14464 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14465 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14466 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14467 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14468 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14469 %}
14470
14471 ins_pipe(pipe_serial);
14472 %}
14473
14474 // ============================================================================
14475 // Compare Instructions
14476
14477 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14478 %{
14479 match(Set cr (CmpI op1 op2));
14480
14481 effect(DEF cr, USE op1, USE op2);
14482
14483 ins_cost(INSN_COST);
14484 format %{ "cmpw $op1, $op2" %}
14485
14486 ins_encode(aarch64_enc_cmpw(op1, op2));
14487
14488 ins_pipe(icmp_reg_reg);
14489 %}
14490
14491 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14492 %{
14493 match(Set cr (CmpI op1 zero));
14494
14495 effect(DEF cr, USE op1);
14496
14497 ins_cost(INSN_COST);
14498 format %{ "cmpw $op1, 0" %}
14499
14500 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14501
14502 ins_pipe(icmp_reg_imm);
14503 %}
14504
14505 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14506 %{
14507 match(Set cr (CmpI op1 op2));
14508
14509 effect(DEF cr, USE op1);
14510
14511 ins_cost(INSN_COST);
14512 format %{ "cmpw $op1, $op2" %}
14513
14514 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14515
14516 ins_pipe(icmp_reg_imm);
14517 %}
14518
14519 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14520 %{
14521 match(Set cr (CmpI op1 op2));
14522
14523 effect(DEF cr, USE op1);
14524
14525 ins_cost(INSN_COST * 2);
14526 format %{ "cmpw $op1, $op2" %}
14527
14528 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14529
14530 ins_pipe(icmp_reg_imm);
14531 %}
14532
14533 // Unsigned compare Instructions; really, same as signed compare
14534 // except it should only be used to feed an If or a CMovI which takes a
14535 // cmpOpU.
14536
14537 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14538 %{
14539 match(Set cr (CmpU op1 op2));
14540
14541 effect(DEF cr, USE op1, USE op2);
14542
14543 ins_cost(INSN_COST);
14544 format %{ "cmpw $op1, $op2\t# unsigned" %}
14545
14546 ins_encode(aarch64_enc_cmpw(op1, op2));
14547
14548 ins_pipe(icmp_reg_reg);
14549 %}
14550
14551 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14552 %{
14553 match(Set cr (CmpU op1 zero));
14554
14555 effect(DEF cr, USE op1);
14556
14557 ins_cost(INSN_COST);
14558 format %{ "cmpw $op1, #0\t# unsigned" %}
14559
14560 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14561
14562 ins_pipe(icmp_reg_imm);
14563 %}
14564
14565 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14566 %{
14567 match(Set cr (CmpU op1 op2));
14568
14569 effect(DEF cr, USE op1);
14570
14571 ins_cost(INSN_COST);
14572 format %{ "cmpw $op1, $op2\t# unsigned" %}
14573
14574 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14575
14576 ins_pipe(icmp_reg_imm);
14577 %}
14578
14579 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14580 %{
14581 match(Set cr (CmpU op1 op2));
14582
14583 effect(DEF cr, USE op1);
14584
14585 ins_cost(INSN_COST * 2);
14586 format %{ "cmpw $op1, $op2\t# unsigned" %}
14587
14588 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14589
14590 ins_pipe(icmp_reg_imm);
14591 %}
14592
14593 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14594 %{
14595 match(Set cr (CmpL op1 op2));
14596
14597 effect(DEF cr, USE op1, USE op2);
14598
14599 ins_cost(INSN_COST);
14600 format %{ "cmp $op1, $op2" %}
14601
14602 ins_encode(aarch64_enc_cmp(op1, op2));
14603
14604 ins_pipe(icmp_reg_reg);
14605 %}
14606
14607 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14608 %{
14609 match(Set cr (CmpL op1 zero));
14610
14611 effect(DEF cr, USE op1);
14612
14613 ins_cost(INSN_COST);
14614 format %{ "tst $op1" %}
14615
14616 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14617
14618 ins_pipe(icmp_reg_imm);
14619 %}
14620
14621 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14622 %{
14623 match(Set cr (CmpL op1 op2));
14624
14625 effect(DEF cr, USE op1);
14626
14627 ins_cost(INSN_COST);
14628 format %{ "cmp $op1, $op2" %}
14629
14630 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14631
14632 ins_pipe(icmp_reg_imm);
14633 %}
14634
14635 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14636 %{
14637 match(Set cr (CmpL op1 op2));
14638
14639 effect(DEF cr, USE op1);
14640
14641 ins_cost(INSN_COST * 2);
14642 format %{ "cmp $op1, $op2" %}
14643
14644 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14645
14646 ins_pipe(icmp_reg_imm);
14647 %}
14648
14649 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14650 %{
14651 match(Set cr (CmpUL op1 op2));
14652
14653 effect(DEF cr, USE op1, USE op2);
14654
14655 ins_cost(INSN_COST);
14656 format %{ "cmp $op1, $op2" %}
14657
14658 ins_encode(aarch64_enc_cmp(op1, op2));
14659
14660 ins_pipe(icmp_reg_reg);
14661 %}
14662
14663 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14664 %{
14665 match(Set cr (CmpUL op1 zero));
14666
14667 effect(DEF cr, USE op1);
14668
14669 ins_cost(INSN_COST);
14670 format %{ "tst $op1" %}
14671
14672 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14673
14674 ins_pipe(icmp_reg_imm);
14675 %}
14676
14677 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14678 %{
14679 match(Set cr (CmpUL op1 op2));
14680
14681 effect(DEF cr, USE op1);
14682
14683 ins_cost(INSN_COST);
14684 format %{ "cmp $op1, $op2" %}
14685
14686 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14687
14688 ins_pipe(icmp_reg_imm);
14689 %}
14690
14691 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14692 %{
14693 match(Set cr (CmpUL op1 op2));
14694
14695 effect(DEF cr, USE op1);
14696
14697 ins_cost(INSN_COST * 2);
14698 format %{ "cmp $op1, $op2" %}
14699
14700 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14701
14702 ins_pipe(icmp_reg_imm);
14703 %}
14704
14705 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14706 %{
14707 match(Set cr (CmpP op1 op2));
14708
14709 effect(DEF cr, USE op1, USE op2);
14710
14711 ins_cost(INSN_COST);
14712 format %{ "cmp $op1, $op2\t // ptr" %}
14713
14714 ins_encode(aarch64_enc_cmpp(op1, op2));
14715
14716 ins_pipe(icmp_reg_reg);
14717 %}
14718
14719 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14720 %{
14721 match(Set cr (CmpN op1 op2));
14722
14723 effect(DEF cr, USE op1, USE op2);
14724
14725 ins_cost(INSN_COST);
14726 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14727
14728 ins_encode(aarch64_enc_cmpn(op1, op2));
14729
14730 ins_pipe(icmp_reg_reg);
14731 %}
14732
14733 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14734 %{
14735 match(Set cr (CmpP op1 zero));
14736
14737 effect(DEF cr, USE op1, USE zero);
14738
14739 ins_cost(INSN_COST);
14740 format %{ "cmp $op1, 0\t // ptr" %}
14741
14742 ins_encode(aarch64_enc_testp(op1));
14743
14744 ins_pipe(icmp_reg_imm);
14745 %}
14746
14747 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14748 %{
14749 match(Set cr (CmpN op1 zero));
14750
14751 effect(DEF cr, USE op1, USE zero);
14752
14753 ins_cost(INSN_COST);
14754 format %{ "cmp $op1, 0\t // compressed ptr" %}
14755
14756 ins_encode(aarch64_enc_testn(op1));
14757
14758 ins_pipe(icmp_reg_imm);
14759 %}
14760
14761 // FP comparisons
14762 //
14763 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14764 // using normal cmpOp. See declaration of rFlagsReg for details.
14765
14766 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14767 %{
14768 match(Set cr (CmpF src1 src2));
14769
14770 ins_cost(3 * INSN_COST);
14771 format %{ "fcmps $src1, $src2" %}
14772
14773 ins_encode %{
14774 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14775 %}
14776
14777 ins_pipe(pipe_class_compare);
14778 %}
14779
14780 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14781 %{
14782 match(Set cr (CmpF src1 src2));
14783
14784 ins_cost(3 * INSN_COST);
14785 format %{ "fcmps $src1, 0.0" %}
14786
14787 ins_encode %{
14788 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14789 %}
14790
14791 ins_pipe(pipe_class_compare);
14792 %}
14793 // FROM HERE
14794
14795 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14796 %{
14797 match(Set cr (CmpD src1 src2));
14798
14799 ins_cost(3 * INSN_COST);
14800 format %{ "fcmpd $src1, $src2" %}
14801
14802 ins_encode %{
14803 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14804 %}
14805
14806 ins_pipe(pipe_class_compare);
14807 %}
14808
14809 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14810 %{
14811 match(Set cr (CmpD src1 src2));
14812
14813 ins_cost(3 * INSN_COST);
14814 format %{ "fcmpd $src1, 0.0" %}
14815
14816 ins_encode %{
14817 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14818 %}
14819
14820 ins_pipe(pipe_class_compare);
14821 %}
14822
14823 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14824 %{
14825 match(Set dst (CmpF3 src1 src2));
14826 effect(KILL cr);
14827
14828 ins_cost(5 * INSN_COST);
14829 format %{ "fcmps $src1, $src2\n\t"
14830 "csinvw($dst, zr, zr, eq\n\t"
14831 "csnegw($dst, $dst, $dst, lt)"
14832 %}
14833
14834 ins_encode %{
14835 Label done;
14836 FloatRegister s1 = as_FloatRegister($src1$$reg);
14837 FloatRegister s2 = as_FloatRegister($src2$$reg);
14838 Register d = as_Register($dst$$reg);
14839 __ fcmps(s1, s2);
14840 // installs 0 if EQ else -1
14841 __ csinvw(d, zr, zr, Assembler::EQ);
14842 // keeps -1 if less or unordered else installs 1
14843 __ csnegw(d, d, d, Assembler::LT);
14844 __ bind(done);
14845 %}
14846
14847 ins_pipe(pipe_class_default);
14848
14849 %}
14850
14851 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14852 %{
14853 match(Set dst (CmpD3 src1 src2));
14854 effect(KILL cr);
14855
14856 ins_cost(5 * INSN_COST);
14857 format %{ "fcmpd $src1, $src2\n\t"
14858 "csinvw($dst, zr, zr, eq\n\t"
14859 "csnegw($dst, $dst, $dst, lt)"
14860 %}
14861
14862 ins_encode %{
14863 Label done;
14864 FloatRegister s1 = as_FloatRegister($src1$$reg);
14865 FloatRegister s2 = as_FloatRegister($src2$$reg);
14866 Register d = as_Register($dst$$reg);
14867 __ fcmpd(s1, s2);
14868 // installs 0 if EQ else -1
14869 __ csinvw(d, zr, zr, Assembler::EQ);
14870 // keeps -1 if less or unordered else installs 1
14871 __ csnegw(d, d, d, Assembler::LT);
14872 __ bind(done);
14873 %}
14874 ins_pipe(pipe_class_default);
14875
14876 %}
14877
14878 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14879 %{
14880 match(Set dst (CmpF3 src1 zero));
14881 effect(KILL cr);
14882
14883 ins_cost(5 * INSN_COST);
14884 format %{ "fcmps $src1, 0.0\n\t"
14885 "csinvw($dst, zr, zr, eq\n\t"
14886 "csnegw($dst, $dst, $dst, lt)"
14887 %}
14888
14889 ins_encode %{
14890 Label done;
14891 FloatRegister s1 = as_FloatRegister($src1$$reg);
14892 Register d = as_Register($dst$$reg);
14893 __ fcmps(s1, 0.0);
14894 // installs 0 if EQ else -1
14895 __ csinvw(d, zr, zr, Assembler::EQ);
14896 // keeps -1 if less or unordered else installs 1
14897 __ csnegw(d, d, d, Assembler::LT);
14898 __ bind(done);
14899 %}
14900
14901 ins_pipe(pipe_class_default);
14902
14903 %}
14904
14905 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14906 %{
14907 match(Set dst (CmpD3 src1 zero));
14908 effect(KILL cr);
14909
14910 ins_cost(5 * INSN_COST);
14911 format %{ "fcmpd $src1, 0.0\n\t"
14912 "csinvw($dst, zr, zr, eq\n\t"
14913 "csnegw($dst, $dst, $dst, lt)"
14914 %}
14915
14916 ins_encode %{
14917 Label done;
14918 FloatRegister s1 = as_FloatRegister($src1$$reg);
14919 Register d = as_Register($dst$$reg);
14920 __ fcmpd(s1, 0.0);
14921 // installs 0 if EQ else -1
14922 __ csinvw(d, zr, zr, Assembler::EQ);
14923 // keeps -1 if less or unordered else installs 1
14924 __ csnegw(d, d, d, Assembler::LT);
14925 __ bind(done);
14926 %}
14927 ins_pipe(pipe_class_default);
14928
14929 %}
14930
14931 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14932 %{
14933 match(Set dst (CmpLTMask p q));
14934 effect(KILL cr);
14935
14936 ins_cost(3 * INSN_COST);
14937
14938 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14939 "csetw $dst, lt\n\t"
14940 "subw $dst, zr, $dst"
14941 %}
14942
14943 ins_encode %{
14944 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14945 __ csetw(as_Register($dst$$reg), Assembler::LT);
14946 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14947 %}
14948
14949 ins_pipe(ialu_reg_reg);
14950 %}
14951
14952 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14953 %{
14954 match(Set dst (CmpLTMask src zero));
14955 effect(KILL cr);
14956
14957 ins_cost(INSN_COST);
14958
14959 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14960
14961 ins_encode %{
14962 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14963 %}
14964
14965 ins_pipe(ialu_reg_shift);
14966 %}
14967
14968 // ============================================================================
14969 // Max and Min
14970
14971 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14972
14973 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14974 %{
14975 effect(DEF cr, USE src);
14976 ins_cost(INSN_COST);
14977 format %{ "cmpw $src, 0" %}
14978
14979 ins_encode %{
14980 __ cmpw($src$$Register, 0);
14981 %}
14982 ins_pipe(icmp_reg_imm);
14983 %}
14984
14985 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14986 %{
14987 match(Set dst (MinI src1 src2));
14988 ins_cost(INSN_COST * 3);
14989
14990 expand %{
14991 rFlagsReg cr;
14992 compI_reg_reg(cr, src1, src2);
14993 cmovI_reg_reg_lt(dst, src1, src2, cr);
14994 %}
14995 %}
14996
14997 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14998 %{
14999 match(Set dst (MaxI src1 src2));
15000 ins_cost(INSN_COST * 3);
15001
15002 expand %{
15003 rFlagsReg cr;
15004 compI_reg_reg(cr, src1, src2);
15005 cmovI_reg_reg_gt(dst, src1, src2, cr);
15006 %}
15007 %}
15008
15009
15010 // ============================================================================
15011 // Branch Instructions
15012
15013 // Direct Branch.
15014 instruct branch(label lbl)
15015 %{
15016 match(Goto);
15017
15018 effect(USE lbl);
15019
15020 ins_cost(BRANCH_COST);
15021 format %{ "b $lbl" %}
15022
15023 ins_encode(aarch64_enc_b(lbl));
15024
15025 ins_pipe(pipe_branch);
15026 %}
15027
15028 // Conditional Near Branch
15029 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15030 %{
15031 // Same match rule as `branchConFar'.
15032 match(If cmp cr);
15033
15034 effect(USE lbl);
15035
15036 ins_cost(BRANCH_COST);
15037 // If set to 1 this indicates that the current instruction is a
15038 // short variant of a long branch. This avoids using this
15039 // instruction in first-pass matching. It will then only be used in
15040 // the `Shorten_branches' pass.
15041 // ins_short_branch(1);
15042 format %{ "b$cmp $lbl" %}
15043
15044 ins_encode(aarch64_enc_br_con(cmp, lbl));
15045
15046 ins_pipe(pipe_branch_cond);
15047 %}
15048
15049 // Conditional Near Branch Unsigned
15050 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15051 %{
15052 // Same match rule as `branchConFar'.
15053 match(If cmp cr);
15054
15055 effect(USE lbl);
15056
15057 ins_cost(BRANCH_COST);
15058 // If set to 1 this indicates that the current instruction is a
15059 // short variant of a long branch. This avoids using this
15060 // instruction in first-pass matching. It will then only be used in
15061 // the `Shorten_branches' pass.
15062 // ins_short_branch(1);
15063 format %{ "b$cmp $lbl\t# unsigned" %}
15064
15065 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15066
15067 ins_pipe(pipe_branch_cond);
15068 %}
15069
15070 // Make use of CBZ and CBNZ. These instructions, as well as being
15071 // shorter than (cmp; branch), have the additional benefit of not
15072 // killing the flags.
15073
15074 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15075 match(If cmp (CmpI op1 op2));
15076 effect(USE labl);
15077
15078 ins_cost(BRANCH_COST);
15079 format %{ "cbw$cmp $op1, $labl" %}
15080 ins_encode %{
15081 Label* L = $labl$$label;
15082 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15083 if (cond == Assembler::EQ)
15084 __ cbzw($op1$$Register, *L);
15085 else
15086 __ cbnzw($op1$$Register, *L);
15087 %}
15088 ins_pipe(pipe_cmp_branch);
15089 %}
15090
15091 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15092 match(If cmp (CmpL op1 op2));
15093 effect(USE labl);
15094
15095 ins_cost(BRANCH_COST);
15096 format %{ "cb$cmp $op1, $labl" %}
15097 ins_encode %{
15098 Label* L = $labl$$label;
15099 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15100 if (cond == Assembler::EQ)
15101 __ cbz($op1$$Register, *L);
15102 else
15103 __ cbnz($op1$$Register, *L);
15104 %}
15105 ins_pipe(pipe_cmp_branch);
15106 %}
15107
15108 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15109 match(If cmp (CmpP op1 op2));
15110 effect(USE labl);
15111
15112 ins_cost(BRANCH_COST);
15113 format %{ "cb$cmp $op1, $labl" %}
15114 ins_encode %{
15115 Label* L = $labl$$label;
15116 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15117 if (cond == Assembler::EQ)
15118 __ cbz($op1$$Register, *L);
15119 else
15120 __ cbnz($op1$$Register, *L);
15121 %}
15122 ins_pipe(pipe_cmp_branch);
15123 %}
15124
15125 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15126 match(If cmp (CmpN op1 op2));
15127 effect(USE labl);
15128
15129 ins_cost(BRANCH_COST);
15130 format %{ "cbw$cmp $op1, $labl" %}
15131 ins_encode %{
15132 Label* L = $labl$$label;
15133 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15134 if (cond == Assembler::EQ)
15135 __ cbzw($op1$$Register, *L);
15136 else
15137 __ cbnzw($op1$$Register, *L);
15138 %}
15139 ins_pipe(pipe_cmp_branch);
15140 %}
15141
15142 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15143 match(If cmp (CmpP (DecodeN oop) zero));
15144 effect(USE labl);
15145
15146 ins_cost(BRANCH_COST);
15147 format %{ "cb$cmp $oop, $labl" %}
15148 ins_encode %{
15149 Label* L = $labl$$label;
15150 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15151 if (cond == Assembler::EQ)
15152 __ cbzw($oop$$Register, *L);
15153 else
15154 __ cbnzw($oop$$Register, *L);
15155 %}
15156 ins_pipe(pipe_cmp_branch);
15157 %}
15158
15159 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15160 match(If cmp (CmpU op1 op2));
15161 effect(USE labl);
15162
15163 ins_cost(BRANCH_COST);
15164 format %{ "cbw$cmp $op1, $labl" %}
15165 ins_encode %{
15166 Label* L = $labl$$label;
15167 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15168 if (cond == Assembler::EQ || cond == Assembler::LS) {
15169 __ cbzw($op1$$Register, *L);
15170 } else {
15171 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15172 __ cbnzw($op1$$Register, *L);
15173 }
15174 %}
15175 ins_pipe(pipe_cmp_branch);
15176 %}
15177
15178 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15179 match(If cmp (CmpUL op1 op2));
15180 effect(USE labl);
15181
15182 ins_cost(BRANCH_COST);
15183 format %{ "cb$cmp $op1, $labl" %}
15184 ins_encode %{
15185 Label* L = $labl$$label;
15186 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15187 if (cond == Assembler::EQ || cond == Assembler::LS) {
15188 __ cbz($op1$$Register, *L);
15189 } else {
15190 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15191 __ cbnz($op1$$Register, *L);
15192 }
15193 %}
15194 ins_pipe(pipe_cmp_branch);
15195 %}
15196
15197 // Test bit and Branch
15198
15199 // Patterns for short (< 32KiB) variants
15200 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15201 match(If cmp (CmpL op1 op2));
15202 effect(USE labl);
15203
15204 ins_cost(BRANCH_COST);
15205 format %{ "cb$cmp $op1, $labl # long" %}
15206 ins_encode %{
15207 Label* L = $labl$$label;
15208 Assembler::Condition cond =
15209 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15210 __ tbr(cond, $op1$$Register, 63, *L);
15211 %}
15212 ins_pipe(pipe_cmp_branch);
15213 ins_short_branch(1);
15214 %}
15215
15216 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15217 match(If cmp (CmpI op1 op2));
15218 effect(USE labl);
15219
15220 ins_cost(BRANCH_COST);
15221 format %{ "cb$cmp $op1, $labl # int" %}
15222 ins_encode %{
15223 Label* L = $labl$$label;
15224 Assembler::Condition cond =
15225 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15226 __ tbr(cond, $op1$$Register, 31, *L);
15227 %}
15228 ins_pipe(pipe_cmp_branch);
15229 ins_short_branch(1);
15230 %}
15231
15232 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15233 match(If cmp (CmpL (AndL op1 op2) op3));
15234 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15235 effect(USE labl);
15236
15237 ins_cost(BRANCH_COST);
15238 format %{ "tb$cmp $op1, $op2, $labl" %}
15239 ins_encode %{
15240 Label* L = $labl$$label;
15241 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15242 int bit = exact_log2_long($op2$$constant);
15243 __ tbr(cond, $op1$$Register, bit, *L);
15244 %}
15245 ins_pipe(pipe_cmp_branch);
15246 ins_short_branch(1);
15247 %}
15248
15249 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15250 match(If cmp (CmpI (AndI op1 op2) op3));
15251 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15252 effect(USE labl);
15253
15254 ins_cost(BRANCH_COST);
15255 format %{ "tb$cmp $op1, $op2, $labl" %}
15256 ins_encode %{
15257 Label* L = $labl$$label;
15258 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15259 int bit = exact_log2((juint)$op2$$constant);
15260 __ tbr(cond, $op1$$Register, bit, *L);
15261 %}
15262 ins_pipe(pipe_cmp_branch);
15263 ins_short_branch(1);
15264 %}
15265
15266 // And far variants
15267 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15268 match(If cmp (CmpL op1 op2));
15269 effect(USE labl);
15270
15271 ins_cost(BRANCH_COST);
15272 format %{ "cb$cmp $op1, $labl # long" %}
15273 ins_encode %{
15274 Label* L = $labl$$label;
15275 Assembler::Condition cond =
15276 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15277 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15278 %}
15279 ins_pipe(pipe_cmp_branch);
15280 %}
15281
15282 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15283 match(If cmp (CmpI op1 op2));
15284 effect(USE labl);
15285
15286 ins_cost(BRANCH_COST);
15287 format %{ "cb$cmp $op1, $labl # int" %}
15288 ins_encode %{
15289 Label* L = $labl$$label;
15290 Assembler::Condition cond =
15291 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15292 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15293 %}
15294 ins_pipe(pipe_cmp_branch);
15295 %}
15296
15297 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15298 match(If cmp (CmpL (AndL op1 op2) op3));
15299 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15300 effect(USE labl);
15301
15302 ins_cost(BRANCH_COST);
15303 format %{ "tb$cmp $op1, $op2, $labl" %}
15304 ins_encode %{
15305 Label* L = $labl$$label;
15306 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15307 int bit = exact_log2_long($op2$$constant);
15308 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15309 %}
15310 ins_pipe(pipe_cmp_branch);
15311 %}
15312
15313 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15314 match(If cmp (CmpI (AndI op1 op2) op3));
15315 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15316 effect(USE labl);
15317
15318 ins_cost(BRANCH_COST);
15319 format %{ "tb$cmp $op1, $op2, $labl" %}
15320 ins_encode %{
15321 Label* L = $labl$$label;
15322 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15323 int bit = exact_log2((juint)$op2$$constant);
15324 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15325 %}
15326 ins_pipe(pipe_cmp_branch);
15327 %}
15328
15329 // Test bits
15330
15331 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15332 match(Set cr (CmpL (AndL op1 op2) op3));
15333 predicate(Assembler::operand_valid_for_logical_immediate
15334 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15335
15336 ins_cost(INSN_COST);
15337 format %{ "tst $op1, $op2 # long" %}
15338 ins_encode %{
15339 __ tst($op1$$Register, $op2$$constant);
15340 %}
15341 ins_pipe(ialu_reg_reg);
15342 %}
15343
15344 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15345 match(Set cr (CmpI (AndI op1 op2) op3));
15346 predicate(Assembler::operand_valid_for_logical_immediate
15347 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15348
15349 ins_cost(INSN_COST);
15350 format %{ "tst $op1, $op2 # int" %}
15351 ins_encode %{
15352 __ tstw($op1$$Register, $op2$$constant);
15353 %}
15354 ins_pipe(ialu_reg_reg);
15355 %}
15356
15357 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15358 match(Set cr (CmpL (AndL op1 op2) op3));
15359
15360 ins_cost(INSN_COST);
15361 format %{ "tst $op1, $op2 # long" %}
15362 ins_encode %{
15363 __ tst($op1$$Register, $op2$$Register);
15364 %}
15365 ins_pipe(ialu_reg_reg);
15366 %}
15367
15368 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15369 match(Set cr (CmpI (AndI op1 op2) op3));
15370
15371 ins_cost(INSN_COST);
15372 format %{ "tstw $op1, $op2 # int" %}
15373 ins_encode %{
15374 __ tstw($op1$$Register, $op2$$Register);
15375 %}
15376 ins_pipe(ialu_reg_reg);
15377 %}
15378
15379
15380 // Conditional Far Branch
15381 // Conditional Far Branch Unsigned
15382 // TODO: fixme
15383
15384 // counted loop end branch near
15385 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15386 %{
15387 match(CountedLoopEnd cmp cr);
15388
15389 effect(USE lbl);
15390
15391 ins_cost(BRANCH_COST);
15392 // short variant.
15393 // ins_short_branch(1);
15394 format %{ "b$cmp $lbl \t// counted loop end" %}
15395
15396 ins_encode(aarch64_enc_br_con(cmp, lbl));
15397
15398 ins_pipe(pipe_branch);
15399 %}
15400
15401 // counted loop end branch far
15402 // TODO: fixme
15403
15404 // ============================================================================
15405 // inlined locking and unlocking
15406
15407 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15408 %{
15409 match(Set cr (FastLock object box));
15410 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15411
15412 ins_cost(5 * INSN_COST);
15413 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15414
15415 ins_encode %{
15416 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15417 %}
15418
15419 ins_pipe(pipe_serial);
15420 %}
15421
15422 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15423 %{
15424 match(Set cr (FastUnlock object box));
15425 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15426
15427 ins_cost(5 * INSN_COST);
15428 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15429
15430 ins_encode %{
15431 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15432 %}
15433
15434 ins_pipe(pipe_serial);
15435 %}
15436
15437 // ============================================================================
15438 // Safepoint Instructions
15439
15440 // TODO
15441 // provide a near and far version of this code
15442
15443 instruct safePoint(rFlagsReg cr, iRegP poll)
15444 %{
15445 match(SafePoint poll);
15446 effect(KILL cr);
15447
15448 format %{
15449 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15450 %}
15451 ins_encode %{
15452 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15453 %}
15454 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15455 %}
15456
15457
15458 // ============================================================================
15459 // Procedure Call/Return Instructions
15460
15461 // Call Java Static Instruction
15462
15463 instruct CallStaticJavaDirect(method meth)
15464 %{
15465 match(CallStaticJava);
15466
15467 effect(USE meth);
15468
15469 ins_cost(CALL_COST);
15470
15471 format %{ "call,static $meth \t// ==> " %}
15472
15473 ins_encode(aarch64_enc_java_static_call(meth),
15474 aarch64_enc_call_epilog);
15475
15476 ins_pipe(pipe_class_call);
15477 %}
15478
15479 // TO HERE
15480
15481 // Call Java Dynamic Instruction
15482 instruct CallDynamicJavaDirect(method meth)
15483 %{
15484 match(CallDynamicJava);
15485
15486 effect(USE meth);
15487
15488 ins_cost(CALL_COST);
15489
15490 format %{ "CALL,dynamic $meth \t// ==> " %}
15491
15492 ins_encode(aarch64_enc_java_dynamic_call(meth),
15493 aarch64_enc_call_epilog);
15494
15495 ins_pipe(pipe_class_call);
15496 %}
15497
15498 // Call Runtime Instruction
15499
15500 instruct CallRuntimeDirect(method meth)
15501 %{
15502 match(CallRuntime);
15503
15504 effect(USE meth);
15505
15506 ins_cost(CALL_COST);
15507
15508 format %{ "CALL, runtime $meth" %}
15509
15510 ins_encode( aarch64_enc_java_to_runtime(meth) );
15511
15512 ins_pipe(pipe_class_call);
15513 %}
15514
15515 // Call Runtime Instruction
15516
15517 instruct CallLeafDirect(method meth)
15518 %{
15519 match(CallLeaf);
15520
15521 effect(USE meth);
15522
15523 ins_cost(CALL_COST);
15524
15525 format %{ "CALL, runtime leaf $meth" %}
15526
15527 ins_encode( aarch64_enc_java_to_runtime(meth) );
15528
15529 ins_pipe(pipe_class_call);
15530 %}
15531
15532 // Call Runtime Instruction without safepoint and with vector arguments
15533 instruct CallLeafDirectVector(method meth)
15534 %{
15535 match(CallLeafVector);
15536
15537 effect(USE meth);
15538
15539 ins_cost(CALL_COST);
15540
15541 format %{ "CALL, runtime leaf vector $meth" %}
15542
15543 ins_encode(aarch64_enc_java_to_runtime(meth));
15544
15545 ins_pipe(pipe_class_call);
15546 %}
15547
15548 // Call Runtime Instruction
15549
15550 // entry point is null, target holds the address to call
15551 instruct CallLeafNoFPIndirect(iRegP target)
15552 %{
15553 predicate(n->as_Call()->entry_point() == nullptr);
15554
15555 match(CallLeafNoFP target);
15556
15557 ins_cost(CALL_COST);
15558
15559 format %{ "CALL, runtime leaf nofp indirect $target" %}
15560
15561 ins_encode %{
15562 __ blr($target$$Register);
15563 %}
15564
15565 ins_pipe(pipe_class_call);
15566 %}
15567
15568 instruct CallLeafNoFPDirect(method meth)
15569 %{
15570 predicate(n->as_Call()->entry_point() != nullptr);
15571
15572 match(CallLeafNoFP);
15573
15574 effect(USE meth);
15575
15576 ins_cost(CALL_COST);
15577
15578 format %{ "CALL, runtime leaf nofp $meth" %}
15579
15580 ins_encode( aarch64_enc_java_to_runtime(meth) );
15581
15582 ins_pipe(pipe_class_call);
15583 %}
15584
15585 // Tail Call; Jump from runtime stub to Java code.
15586 // Also known as an 'interprocedural jump'.
15587 // Target of jump will eventually return to caller.
15588 // TailJump below removes the return address.
15589 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15590 // emitted just above the TailCall which has reset rfp to the caller state.
15591 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15592 %{
15593 match(TailCall jump_target method_ptr);
15594
15595 ins_cost(CALL_COST);
15596
15597 format %{ "br $jump_target\t# $method_ptr holds method" %}
15598
15599 ins_encode(aarch64_enc_tail_call(jump_target));
15600
15601 ins_pipe(pipe_class_call);
15602 %}
15603
15604 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15605 %{
15606 match(TailJump jump_target ex_oop);
15607
15608 ins_cost(CALL_COST);
15609
15610 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15611
15612 ins_encode(aarch64_enc_tail_jmp(jump_target));
15613
15614 ins_pipe(pipe_class_call);
15615 %}
15616
15617 // Forward exception.
15618 instruct ForwardExceptionjmp()
15619 %{
15620 match(ForwardException);
15621 ins_cost(CALL_COST);
15622
15623 format %{ "b forward_exception_stub" %}
15624 ins_encode %{
15625 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15626 %}
15627 ins_pipe(pipe_class_call);
15628 %}
15629
15630 // Create exception oop: created by stack-crawling runtime code.
15631 // Created exception is now available to this handler, and is setup
15632 // just prior to jumping to this handler. No code emitted.
15633 // TODO check
15634 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15635 instruct CreateException(iRegP_R0 ex_oop)
15636 %{
15637 match(Set ex_oop (CreateEx));
15638
15639 format %{ " -- \t// exception oop; no code emitted" %}
15640
15641 size(0);
15642
15643 ins_encode( /*empty*/ );
15644
15645 ins_pipe(pipe_class_empty);
15646 %}
15647
15648 // Rethrow exception: The exception oop will come in the first
15649 // argument position. Then JUMP (not call) to the rethrow stub code.
15650 instruct RethrowException() %{
15651 match(Rethrow);
15652 ins_cost(CALL_COST);
15653
15654 format %{ "b rethrow_stub" %}
15655
15656 ins_encode( aarch64_enc_rethrow() );
15657
15658 ins_pipe(pipe_class_call);
15659 %}
15660
15661
15662 // Return Instruction
15663 // epilog node loads ret address into lr as part of frame pop
15664 instruct Ret()
15665 %{
15666 match(Return);
15667
15668 format %{ "ret\t// return register" %}
15669
15670 ins_encode( aarch64_enc_ret() );
15671
15672 ins_pipe(pipe_branch);
15673 %}
15674
15675 // Die now.
15676 instruct ShouldNotReachHere() %{
15677 match(Halt);
15678
15679 ins_cost(CALL_COST);
15680 format %{ "ShouldNotReachHere" %}
15681
15682 ins_encode %{
15683 if (is_reachable()) {
15684 const char* str = __ code_string(_halt_reason);
15685 __ stop(str);
15686 }
15687 %}
15688
15689 ins_pipe(pipe_class_default);
15690 %}
15691
15692 // ============================================================================
15693 // Partial Subtype Check
15694 //
15695 // superklass array for an instance of the superklass. Set a hidden
15696 // internal cache on a hit (cache is checked with exposed code in
15697 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15698 // encoding ALSO sets flags.
15699
15700 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15701 %{
15702 match(Set result (PartialSubtypeCheck sub super));
15703 predicate(!UseSecondarySupersTable);
15704 effect(KILL cr, KILL temp);
15705
15706 ins_cost(20 * INSN_COST); // slightly larger than the next version
15707 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15708
15709 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15710
15711 opcode(0x1); // Force zero of result reg on hit
15712
15713 ins_pipe(pipe_class_memory);
15714 %}
15715
15716 // Two versions of partialSubtypeCheck, both used when we need to
15717 // search for a super class in the secondary supers array. The first
15718 // is used when we don't know _a priori_ the class being searched
15719 // for. The second, far more common, is used when we do know: this is
15720 // used for instanceof, checkcast, and any case where C2 can determine
15721 // it by constant propagation.
15722
15723 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15724 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15725 rFlagsReg cr)
15726 %{
15727 match(Set result (PartialSubtypeCheck sub super));
15728 predicate(UseSecondarySupersTable);
15729 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15730
15731 ins_cost(10 * INSN_COST); // slightly larger than the next version
15732 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15733
15734 ins_encode %{
15735 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15736 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15737 $vtemp$$FloatRegister,
15738 $result$$Register, /*L_success*/nullptr);
15739 %}
15740
15741 ins_pipe(pipe_class_memory);
15742 %}
15743
15744 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15745 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15746 rFlagsReg cr)
15747 %{
15748 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15749 predicate(UseSecondarySupersTable);
15750 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15751
15752 ins_cost(5 * INSN_COST); // smaller than the next version
15753 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15754
15755 ins_encode %{
15756 bool success = false;
15757 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15758 if (InlineSecondarySupersTest) {
15759 success =
15760 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15761 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15762 $vtemp$$FloatRegister,
15763 $result$$Register,
15764 super_klass_slot);
15765 } else {
15766 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15767 success = (call != nullptr);
15768 }
15769 if (!success) {
15770 ciEnv::current()->record_failure("CodeCache is full");
15771 return;
15772 }
15773 %}
15774
15775 ins_pipe(pipe_class_memory);
15776 %}
15777
15778 // Intrisics for String.compareTo()
15779
15780 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15781 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15782 %{
15783 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15784 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15785 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15786
15787 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15788 ins_encode %{
15789 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15790 __ string_compare($str1$$Register, $str2$$Register,
15791 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15792 $tmp1$$Register, $tmp2$$Register,
15793 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15794 %}
15795 ins_pipe(pipe_class_memory);
15796 %}
15797
15798 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15799 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15800 %{
15801 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15802 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15803 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15804
15805 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15806 ins_encode %{
15807 __ string_compare($str1$$Register, $str2$$Register,
15808 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15809 $tmp1$$Register, $tmp2$$Register,
15810 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15811 %}
15812 ins_pipe(pipe_class_memory);
15813 %}
15814
15815 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15816 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15817 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15818 %{
15819 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15820 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15821 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15822 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15823
15824 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15825 ins_encode %{
15826 __ string_compare($str1$$Register, $str2$$Register,
15827 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15828 $tmp1$$Register, $tmp2$$Register,
15829 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15830 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15831 %}
15832 ins_pipe(pipe_class_memory);
15833 %}
15834
15835 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15836 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15837 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15838 %{
15839 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15840 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15841 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15842 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15843
15844 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15845 ins_encode %{
15846 __ string_compare($str1$$Register, $str2$$Register,
15847 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15848 $tmp1$$Register, $tmp2$$Register,
15849 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15850 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15851 %}
15852 ins_pipe(pipe_class_memory);
15853 %}
15854
15855 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15856 // these string_compare variants as NEON register type for convenience so that the prototype of
15857 // string_compare can be shared with all variants.
15858
15859 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15860 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15861 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15862 pRegGov_P1 pgtmp2, rFlagsReg cr)
15863 %{
15864 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15865 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15866 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15867 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15868
15869 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15870 ins_encode %{
15871 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15872 __ string_compare($str1$$Register, $str2$$Register,
15873 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15874 $tmp1$$Register, $tmp2$$Register,
15875 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15876 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15877 StrIntrinsicNode::LL);
15878 %}
15879 ins_pipe(pipe_class_memory);
15880 %}
15881
15882 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15883 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15884 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15885 pRegGov_P1 pgtmp2, rFlagsReg cr)
15886 %{
15887 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15888 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15889 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15890 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15891
15892 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15893 ins_encode %{
15894 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15895 __ string_compare($str1$$Register, $str2$$Register,
15896 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15897 $tmp1$$Register, $tmp2$$Register,
15898 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15899 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15900 StrIntrinsicNode::LU);
15901 %}
15902 ins_pipe(pipe_class_memory);
15903 %}
15904
15905 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15906 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15907 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15908 pRegGov_P1 pgtmp2, rFlagsReg cr)
15909 %{
15910 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15911 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15912 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15913 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15914
15915 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15916 ins_encode %{
15917 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15918 __ string_compare($str1$$Register, $str2$$Register,
15919 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15920 $tmp1$$Register, $tmp2$$Register,
15921 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15922 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15923 StrIntrinsicNode::UL);
15924 %}
15925 ins_pipe(pipe_class_memory);
15926 %}
15927
15928 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15929 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15930 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15931 pRegGov_P1 pgtmp2, rFlagsReg cr)
15932 %{
15933 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15934 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15935 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15936 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15937
15938 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15939 ins_encode %{
15940 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15941 __ string_compare($str1$$Register, $str2$$Register,
15942 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15943 $tmp1$$Register, $tmp2$$Register,
15944 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15945 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15946 StrIntrinsicNode::UU);
15947 %}
15948 ins_pipe(pipe_class_memory);
15949 %}
15950
15951 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15952 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15953 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15954 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15955 %{
15956 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15957 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15958 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15959 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15960 TEMP vtmp0, TEMP vtmp1, KILL cr);
15961 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15962 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15963
15964 ins_encode %{
15965 __ string_indexof($str1$$Register, $str2$$Register,
15966 $cnt1$$Register, $cnt2$$Register,
15967 $tmp1$$Register, $tmp2$$Register,
15968 $tmp3$$Register, $tmp4$$Register,
15969 $tmp5$$Register, $tmp6$$Register,
15970 -1, $result$$Register, StrIntrinsicNode::UU);
15971 %}
15972 ins_pipe(pipe_class_memory);
15973 %}
15974
15975 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15976 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15977 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15978 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15979 %{
15980 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15981 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15982 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15983 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15984 TEMP vtmp0, TEMP vtmp1, KILL cr);
15985 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15986 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15987
15988 ins_encode %{
15989 __ string_indexof($str1$$Register, $str2$$Register,
15990 $cnt1$$Register, $cnt2$$Register,
15991 $tmp1$$Register, $tmp2$$Register,
15992 $tmp3$$Register, $tmp4$$Register,
15993 $tmp5$$Register, $tmp6$$Register,
15994 -1, $result$$Register, StrIntrinsicNode::LL);
15995 %}
15996 ins_pipe(pipe_class_memory);
15997 %}
15998
15999 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16000 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16001 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16002 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16003 %{
16004 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16005 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16006 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16007 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16008 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16009 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16010 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16011
16012 ins_encode %{
16013 __ string_indexof($str1$$Register, $str2$$Register,
16014 $cnt1$$Register, $cnt2$$Register,
16015 $tmp1$$Register, $tmp2$$Register,
16016 $tmp3$$Register, $tmp4$$Register,
16017 $tmp5$$Register, $tmp6$$Register,
16018 -1, $result$$Register, StrIntrinsicNode::UL);
16019 %}
16020 ins_pipe(pipe_class_memory);
16021 %}
16022
16023 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16024 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16025 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16026 %{
16027 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16028 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16029 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16030 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16031 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16032 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16033
16034 ins_encode %{
16035 int icnt2 = (int)$int_cnt2$$constant;
16036 __ string_indexof($str1$$Register, $str2$$Register,
16037 $cnt1$$Register, zr,
16038 $tmp1$$Register, $tmp2$$Register,
16039 $tmp3$$Register, $tmp4$$Register, zr, zr,
16040 icnt2, $result$$Register, StrIntrinsicNode::UU);
16041 %}
16042 ins_pipe(pipe_class_memory);
16043 %}
16044
16045 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16046 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16047 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16048 %{
16049 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16050 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16051 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16052 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16053 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16054 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16055
16056 ins_encode %{
16057 int icnt2 = (int)$int_cnt2$$constant;
16058 __ string_indexof($str1$$Register, $str2$$Register,
16059 $cnt1$$Register, zr,
16060 $tmp1$$Register, $tmp2$$Register,
16061 $tmp3$$Register, $tmp4$$Register, zr, zr,
16062 icnt2, $result$$Register, StrIntrinsicNode::LL);
16063 %}
16064 ins_pipe(pipe_class_memory);
16065 %}
16066
16067 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16068 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16069 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16070 %{
16071 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16072 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16073 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16074 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16075 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16076 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16077
16078 ins_encode %{
16079 int icnt2 = (int)$int_cnt2$$constant;
16080 __ string_indexof($str1$$Register, $str2$$Register,
16081 $cnt1$$Register, zr,
16082 $tmp1$$Register, $tmp2$$Register,
16083 $tmp3$$Register, $tmp4$$Register, zr, zr,
16084 icnt2, $result$$Register, StrIntrinsicNode::UL);
16085 %}
16086 ins_pipe(pipe_class_memory);
16087 %}
16088
16089 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16090 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16091 iRegINoSp tmp3, rFlagsReg cr)
16092 %{
16093 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16094 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16095 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16096 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16097
16098 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16099
16100 ins_encode %{
16101 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16102 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16103 $tmp3$$Register);
16104 %}
16105 ins_pipe(pipe_class_memory);
16106 %}
16107
16108 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16109 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16110 iRegINoSp tmp3, rFlagsReg cr)
16111 %{
16112 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16113 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16114 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16115 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16116
16117 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16118
16119 ins_encode %{
16120 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16121 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16122 $tmp3$$Register);
16123 %}
16124 ins_pipe(pipe_class_memory);
16125 %}
16126
16127 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16128 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16129 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16130 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16131 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16132 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16133 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16134 ins_encode %{
16135 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16136 $result$$Register, $ztmp1$$FloatRegister,
16137 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16138 $ptmp$$PRegister, true /* isL */);
16139 %}
16140 ins_pipe(pipe_class_memory);
16141 %}
16142
16143 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16144 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16145 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16146 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16147 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16148 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16149 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16150 ins_encode %{
16151 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16152 $result$$Register, $ztmp1$$FloatRegister,
16153 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16154 $ptmp$$PRegister, false /* isL */);
16155 %}
16156 ins_pipe(pipe_class_memory);
16157 %}
16158
16159 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16160 iRegI_R0 result, rFlagsReg cr)
16161 %{
16162 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16163 match(Set result (StrEquals (Binary str1 str2) cnt));
16164 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16165
16166 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16167 ins_encode %{
16168 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16169 __ string_equals($str1$$Register, $str2$$Register,
16170 $result$$Register, $cnt$$Register);
16171 %}
16172 ins_pipe(pipe_class_memory);
16173 %}
16174
16175 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16176 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16177 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16178 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16179 iRegP_R10 tmp, rFlagsReg cr)
16180 %{
16181 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16182 match(Set result (AryEq ary1 ary2));
16183 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16184 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16185 TEMP vtmp6, TEMP vtmp7, KILL cr);
16186
16187 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16188 ins_encode %{
16189 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16190 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16191 $result$$Register, $tmp$$Register, 1);
16192 if (tpc == nullptr) {
16193 ciEnv::current()->record_failure("CodeCache is full");
16194 return;
16195 }
16196 %}
16197 ins_pipe(pipe_class_memory);
16198 %}
16199
16200 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16201 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16202 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16203 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16204 iRegP_R10 tmp, rFlagsReg cr)
16205 %{
16206 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16207 match(Set result (AryEq ary1 ary2));
16208 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16209 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16210 TEMP vtmp6, TEMP vtmp7, KILL cr);
16211
16212 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16213 ins_encode %{
16214 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16215 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16216 $result$$Register, $tmp$$Register, 2);
16217 if (tpc == nullptr) {
16218 ciEnv::current()->record_failure("CodeCache is full");
16219 return;
16220 }
16221 %}
16222 ins_pipe(pipe_class_memory);
16223 %}
16224
16225 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16226 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16227 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16228 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16229 %{
16230 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16231 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16232 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16233
16234 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16235 ins_encode %{
16236 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16237 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16238 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16239 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16240 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16241 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16242 (BasicType)$basic_type$$constant);
16243 if (tpc == nullptr) {
16244 ciEnv::current()->record_failure("CodeCache is full");
16245 return;
16246 }
16247 %}
16248 ins_pipe(pipe_class_memory);
16249 %}
16250
16251 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16252 %{
16253 match(Set result (CountPositives ary1 len));
16254 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16255 format %{ "count positives byte[] $ary1,$len -> $result" %}
16256 ins_encode %{
16257 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16258 if (tpc == nullptr) {
16259 ciEnv::current()->record_failure("CodeCache is full");
16260 return;
16261 }
16262 %}
16263 ins_pipe( pipe_slow );
16264 %}
16265
16266 // fast char[] to byte[] compression
16267 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16268 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16269 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16270 iRegI_R0 result, rFlagsReg cr)
16271 %{
16272 match(Set result (StrCompressedCopy src (Binary dst len)));
16273 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16274 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16275
16276 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16277 ins_encode %{
16278 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16279 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16280 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16281 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16282 %}
16283 ins_pipe(pipe_slow);
16284 %}
16285
16286 // fast byte[] to char[] inflation
16287 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16288 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16289 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16290 %{
16291 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16292 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16293 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16294 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16295
16296 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16297 ins_encode %{
16298 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16299 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16300 $vtmp2$$FloatRegister, $tmp$$Register);
16301 if (tpc == nullptr) {
16302 ciEnv::current()->record_failure("CodeCache is full");
16303 return;
16304 }
16305 %}
16306 ins_pipe(pipe_class_memory);
16307 %}
16308
16309 // encode char[] to byte[] in ISO_8859_1
16310 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16311 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16312 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16313 iRegI_R0 result, rFlagsReg cr)
16314 %{
16315 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16316 match(Set result (EncodeISOArray src (Binary dst len)));
16317 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16318 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16319
16320 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16321 ins_encode %{
16322 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16323 $result$$Register, false,
16324 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16325 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16326 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16327 %}
16328 ins_pipe(pipe_class_memory);
16329 %}
16330
16331 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16332 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16333 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16334 iRegI_R0 result, rFlagsReg cr)
16335 %{
16336 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16337 match(Set result (EncodeISOArray src (Binary dst len)));
16338 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16339 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16340
16341 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16342 ins_encode %{
16343 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16344 $result$$Register, true,
16345 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16346 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16347 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16348 %}
16349 ins_pipe(pipe_class_memory);
16350 %}
16351
16352 //----------------------------- CompressBits/ExpandBits ------------------------
16353
16354 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16355 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16356 match(Set dst (CompressBits src mask));
16357 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16358 format %{ "mov $tsrc, $src\n\t"
16359 "mov $tmask, $mask\n\t"
16360 "bext $tdst, $tsrc, $tmask\n\t"
16361 "mov $dst, $tdst"
16362 %}
16363 ins_encode %{
16364 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16365 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16366 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16367 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16368 %}
16369 ins_pipe(pipe_slow);
16370 %}
16371
16372 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16373 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16374 match(Set dst (CompressBits (LoadI mem) mask));
16375 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16376 format %{ "ldrs $tsrc, $mem\n\t"
16377 "ldrs $tmask, $mask\n\t"
16378 "bext $tdst, $tsrc, $tmask\n\t"
16379 "mov $dst, $tdst"
16380 %}
16381 ins_encode %{
16382 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16383 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16384 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16385 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16386 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16387 %}
16388 ins_pipe(pipe_slow);
16389 %}
16390
16391 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16392 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16393 match(Set dst (CompressBits src mask));
16394 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16395 format %{ "mov $tsrc, $src\n\t"
16396 "mov $tmask, $mask\n\t"
16397 "bext $tdst, $tsrc, $tmask\n\t"
16398 "mov $dst, $tdst"
16399 %}
16400 ins_encode %{
16401 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16402 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16403 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16404 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16405 %}
16406 ins_pipe(pipe_slow);
16407 %}
16408
16409 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16410 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16411 match(Set dst (CompressBits (LoadL mem) mask));
16412 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16413 format %{ "ldrd $tsrc, $mem\n\t"
16414 "ldrd $tmask, $mask\n\t"
16415 "bext $tdst, $tsrc, $tmask\n\t"
16416 "mov $dst, $tdst"
16417 %}
16418 ins_encode %{
16419 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16420 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16421 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16422 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16423 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16424 %}
16425 ins_pipe(pipe_slow);
16426 %}
16427
16428 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16429 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16430 match(Set dst (ExpandBits src mask));
16431 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16432 format %{ "mov $tsrc, $src\n\t"
16433 "mov $tmask, $mask\n\t"
16434 "bdep $tdst, $tsrc, $tmask\n\t"
16435 "mov $dst, $tdst"
16436 %}
16437 ins_encode %{
16438 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16439 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16440 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16441 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16442 %}
16443 ins_pipe(pipe_slow);
16444 %}
16445
16446 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16447 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16448 match(Set dst (ExpandBits (LoadI mem) mask));
16449 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16450 format %{ "ldrs $tsrc, $mem\n\t"
16451 "ldrs $tmask, $mask\n\t"
16452 "bdep $tdst, $tsrc, $tmask\n\t"
16453 "mov $dst, $tdst"
16454 %}
16455 ins_encode %{
16456 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16457 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16458 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16459 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16460 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16461 %}
16462 ins_pipe(pipe_slow);
16463 %}
16464
16465 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16466 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16467 match(Set dst (ExpandBits src mask));
16468 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16469 format %{ "mov $tsrc, $src\n\t"
16470 "mov $tmask, $mask\n\t"
16471 "bdep $tdst, $tsrc, $tmask\n\t"
16472 "mov $dst, $tdst"
16473 %}
16474 ins_encode %{
16475 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16476 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16477 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16478 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16479 %}
16480 ins_pipe(pipe_slow);
16481 %}
16482
16483
16484 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16485 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16486 match(Set dst (ExpandBits (LoadL mem) mask));
16487 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16488 format %{ "ldrd $tsrc, $mem\n\t"
16489 "ldrd $tmask, $mask\n\t"
16490 "bdep $tdst, $tsrc, $tmask\n\t"
16491 "mov $dst, $tdst"
16492 %}
16493 ins_encode %{
16494 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16495 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16496 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16497 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16498 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16499 %}
16500 ins_pipe(pipe_slow);
16501 %}
16502
16503 //----------------------------- Reinterpret ----------------------------------
16504 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16505 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16506 match(Set dst (ReinterpretHF2S src));
16507 format %{ "reinterpretHF2S $dst, $src" %}
16508 ins_encode %{
16509 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16510 %}
16511 ins_pipe(pipe_slow);
16512 %}
16513
16514 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16515 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16516 match(Set dst (ReinterpretS2HF src));
16517 format %{ "reinterpretS2HF $dst, $src" %}
16518 ins_encode %{
16519 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16520 %}
16521 ins_pipe(pipe_slow);
16522 %}
16523
16524 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16525 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16526 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16527 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16528 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16529 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16530 // can be omitted in this pattern, resulting in -
16531 // fcvt $dst, $src // Convert float to half-precision float
16532 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16533 %{
16534 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16535 format %{ "convF2HFAndS2HF $dst, $src" %}
16536 ins_encode %{
16537 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16538 %}
16539 ins_pipe(pipe_slow);
16540 %}
16541
16542 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16543 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16544 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16545 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16546 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16547 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16548 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16549 // resulting in -
16550 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16551 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16552 %{
16553 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16554 format %{ "convHF2SAndHF2F $dst, $src" %}
16555 ins_encode %{
16556 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16557 %}
16558 ins_pipe(pipe_slow);
16559 %}
16560
16561 // ============================================================================
16562 // This name is KNOWN by the ADLC and cannot be changed.
16563 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16564 // for this guy.
16565 instruct tlsLoadP(thread_RegP dst)
16566 %{
16567 match(Set dst (ThreadLocal));
16568
16569 ins_cost(0);
16570
16571 format %{ " -- \t// $dst=Thread::current(), empty" %}
16572
16573 size(0);
16574
16575 ins_encode( /*empty*/ );
16576
16577 ins_pipe(pipe_class_empty);
16578 %}
16579
16580 //----------PEEPHOLE RULES-----------------------------------------------------
16581 // These must follow all instruction definitions as they use the names
16582 // defined in the instructions definitions.
16583 //
16584 // peepmatch ( root_instr_name [preceding_instruction]* );
16585 //
16586 // peepconstraint %{
16587 // (instruction_number.operand_name relational_op instruction_number.operand_name
16588 // [, ...] );
16589 // // instruction numbers are zero-based using left to right order in peepmatch
16590 //
16591 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16592 // // provide an instruction_number.operand_name for each operand that appears
16593 // // in the replacement instruction's match rule
16594 //
16595 // ---------VM FLAGS---------------------------------------------------------
16596 //
16597 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16598 //
16599 // Each peephole rule is given an identifying number starting with zero and
16600 // increasing by one in the order seen by the parser. An individual peephole
16601 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16602 // on the command-line.
16603 //
16604 // ---------CURRENT LIMITATIONS----------------------------------------------
16605 //
16606 // Only match adjacent instructions in same basic block
16607 // Only equality constraints
16608 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16609 // Only one replacement instruction
16610 //
16611 // ---------EXAMPLE----------------------------------------------------------
16612 //
16613 // // pertinent parts of existing instructions in architecture description
16614 // instruct movI(iRegINoSp dst, iRegI src)
16615 // %{
16616 // match(Set dst (CopyI src));
16617 // %}
16618 //
16619 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16620 // %{
16621 // match(Set dst (AddI dst src));
16622 // effect(KILL cr);
16623 // %}
16624 //
16625 // // Change (inc mov) to lea
16626 // peephole %{
16627 // // increment preceded by register-register move
16628 // peepmatch ( incI_iReg movI );
16629 // // require that the destination register of the increment
16630 // // match the destination register of the move
16631 // peepconstraint ( 0.dst == 1.dst );
16632 // // construct a replacement instruction that sets
16633 // // the destination to ( move's source register + one )
16634 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16635 // %}
16636 //
16637
16638 // Implementation no longer uses movX instructions since
16639 // machine-independent system no longer uses CopyX nodes.
16640 //
16641 // peephole
16642 // %{
16643 // peepmatch (incI_iReg movI);
16644 // peepconstraint (0.dst == 1.dst);
16645 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16646 // %}
16647
16648 // peephole
16649 // %{
16650 // peepmatch (decI_iReg movI);
16651 // peepconstraint (0.dst == 1.dst);
16652 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16653 // %}
16654
16655 // peephole
16656 // %{
16657 // peepmatch (addI_iReg_imm movI);
16658 // peepconstraint (0.dst == 1.dst);
16659 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16660 // %}
16661
16662 // peephole
16663 // %{
16664 // peepmatch (incL_iReg movL);
16665 // peepconstraint (0.dst == 1.dst);
16666 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16667 // %}
16668
16669 // peephole
16670 // %{
16671 // peepmatch (decL_iReg movL);
16672 // peepconstraint (0.dst == 1.dst);
16673 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16674 // %}
16675
16676 // peephole
16677 // %{
16678 // peepmatch (addL_iReg_imm movL);
16679 // peepconstraint (0.dst == 1.dst);
16680 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16681 // %}
16682
16683 // peephole
16684 // %{
16685 // peepmatch (addP_iReg_imm movP);
16686 // peepconstraint (0.dst == 1.dst);
16687 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16688 // %}
16689
16690 // // Change load of spilled value to only a spill
16691 // instruct storeI(memory mem, iRegI src)
16692 // %{
16693 // match(Set mem (StoreI mem src));
16694 // %}
16695 //
16696 // instruct loadI(iRegINoSp dst, memory mem)
16697 // %{
16698 // match(Set dst (LoadI mem));
16699 // %}
16700 //
16701
16702 //----------SMARTSPILL RULES---------------------------------------------------
16703 // These must follow all instruction definitions as they use the names
16704 // defined in the instructions definitions.
16705
16706 // Local Variables:
16707 // mode: c++
16708 // End: