1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 bool use_far_branch = MacroAssembler::target_needs_far_branch(SharedRuntime::deopt_blob()->unpack());
1202 // far: adrp, add, blr; near: bl
1203 uint target_branch_instructions = use_far_branch ? 3 : 1;
1204 // target branch + one branch instruction
1205 uint deopt_handler_instructions = target_branch_instructions + 1;
1206 return deopt_handler_instructions * NativeInstruction::instruction_size;
1207 }
1208 };
1209
1210 class Node::PD {
1211 public:
1212 enum NodeFlags {
1213 _last_flag = Node::_last_flag
1214 };
1215 };
1216
1217 bool is_CAS(int opcode, bool maybe_volatile);
1218
1219 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1220
1221 bool unnecessary_acquire(const Node *barrier);
1222 bool needs_acquiring_load(const Node *load);
1223
1224 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1225
1226 bool unnecessary_release(const Node *barrier);
1227 bool unnecessary_volatile(const Node *barrier);
1228 bool needs_releasing_store(const Node *store);
1229
1230 // predicate controlling translation of CompareAndSwapX
1231 bool needs_acquiring_load_exclusive(const Node *load);
1232
1233 // predicate controlling addressing modes
1234 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1235
1236 // Convert BoolTest condition to Assembler condition.
1237 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1238 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1239 %}
1240
1241 source %{
1242
1243 // Derived RegMask with conditionally allocatable registers
1244
1245 void PhaseOutput::pd_perform_mach_node_analysis() {
1246 }
1247
1248 int MachNode::pd_alignment_required() const {
1249 return 1;
1250 }
1251
1252 int MachNode::compute_padding(int current_offset) const {
1253 return 0;
1254 }
1255
1256 RegMask _ANY_REG32_mask;
1257 RegMask _ANY_REG_mask;
1258 RegMask _PTR_REG_mask;
1259 RegMask _NO_SPECIAL_REG32_mask;
1260 RegMask _NO_SPECIAL_REG_mask;
1261 RegMask _NO_SPECIAL_PTR_REG_mask;
1262 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1263
1264 void reg_mask_init() {
1265 // We derive below RegMask(s) from the ones which are auto-generated from
1266 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1267 // registers conditionally reserved.
1268
1269 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1270 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1271
1272 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1273
1274 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1275
1276 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1277 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1278
1279 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1280 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1281
1282 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1283 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1284
1285 // r27 is not allocatable when compressed oops is on and heapbase is not
1286 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1287 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1288 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1289 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1290 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1291 }
1292
1293 // r29 is not allocatable when PreserveFramePointer is on
1294 if (PreserveFramePointer) {
1295 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1296 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1297 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1301 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1302 }
1303
1304 // Optimizaton of volatile gets and puts
1305 // -------------------------------------
1306 //
1307 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1308 // use to implement volatile reads and writes. For a volatile read
1309 // we simply need
1310 //
1311 // ldar<x>
1312 //
1313 // and for a volatile write we need
1314 //
1315 // stlr<x>
1316 //
1317 // Alternatively, we can implement them by pairing a normal
1318 // load/store with a memory barrier. For a volatile read we need
1319 //
1320 // ldr<x>
1321 // dmb ishld
1322 //
1323 // for a volatile write
1324 //
1325 // dmb ish
1326 // str<x>
1327 // dmb ish
1328 //
1329 // We can also use ldaxr and stlxr to implement compare and swap CAS
1330 // sequences. These are normally translated to an instruction
1331 // sequence like the following
1332 //
1333 // dmb ish
1334 // retry:
1335 // ldxr<x> rval raddr
1336 // cmp rval rold
1337 // b.ne done
1338 // stlxr<x> rval, rnew, rold
1339 // cbnz rval retry
1340 // done:
1341 // cset r0, eq
1342 // dmb ishld
1343 //
1344 // Note that the exclusive store is already using an stlxr
1345 // instruction. That is required to ensure visibility to other
1346 // threads of the exclusive write (assuming it succeeds) before that
1347 // of any subsequent writes.
1348 //
1349 // The following instruction sequence is an improvement on the above
1350 //
1351 // retry:
1352 // ldaxr<x> rval raddr
1353 // cmp rval rold
1354 // b.ne done
1355 // stlxr<x> rval, rnew, rold
1356 // cbnz rval retry
1357 // done:
1358 // cset r0, eq
1359 //
1360 // We don't need the leading dmb ish since the stlxr guarantees
1361 // visibility of prior writes in the case that the swap is
1362 // successful. Crucially we don't have to worry about the case where
1363 // the swap is not successful since no valid program should be
1364 // relying on visibility of prior changes by the attempting thread
1365 // in the case where the CAS fails.
1366 //
1367 // Similarly, we don't need the trailing dmb ishld if we substitute
1368 // an ldaxr instruction since that will provide all the guarantees we
1369 // require regarding observation of changes made by other threads
1370 // before any change to the CAS address observed by the load.
1371 //
1372 // In order to generate the desired instruction sequence we need to
1373 // be able to identify specific 'signature' ideal graph node
1374 // sequences which i) occur as a translation of a volatile reads or
1375 // writes or CAS operations and ii) do not occur through any other
1376 // translation or graph transformation. We can then provide
1377 // alternative aldc matching rules which translate these node
1378 // sequences to the desired machine code sequences. Selection of the
1379 // alternative rules can be implemented by predicates which identify
1380 // the relevant node sequences.
1381 //
1382 // The ideal graph generator translates a volatile read to the node
1383 // sequence
1384 //
1385 // LoadX[mo_acquire]
1386 // MemBarAcquire
1387 //
1388 // As a special case when using the compressed oops optimization we
1389 // may also see this variant
1390 //
1391 // LoadN[mo_acquire]
1392 // DecodeN
1393 // MemBarAcquire
1394 //
1395 // A volatile write is translated to the node sequence
1396 //
1397 // MemBarRelease
1398 // StoreX[mo_release] {CardMark}-optional
1399 // MemBarVolatile
1400 //
1401 // n.b. the above node patterns are generated with a strict
1402 // 'signature' configuration of input and output dependencies (see
1403 // the predicates below for exact details). The card mark may be as
1404 // simple as a few extra nodes or, in a few GC configurations, may
1405 // include more complex control flow between the leading and
1406 // trailing memory barriers. However, whatever the card mark
1407 // configuration these signatures are unique to translated volatile
1408 // reads/stores -- they will not appear as a result of any other
1409 // bytecode translation or inlining nor as a consequence of
1410 // optimizing transforms.
1411 //
1412 // We also want to catch inlined unsafe volatile gets and puts and
1413 // be able to implement them using either ldar<x>/stlr<x> or some
1414 // combination of ldr<x>/stlr<x> and dmb instructions.
1415 //
1416 // Inlined unsafe volatiles puts manifest as a minor variant of the
1417 // normal volatile put node sequence containing an extra cpuorder
1418 // membar
1419 //
1420 // MemBarRelease
1421 // MemBarCPUOrder
1422 // StoreX[mo_release] {CardMark}-optional
1423 // MemBarCPUOrder
1424 // MemBarVolatile
1425 //
1426 // n.b. as an aside, a cpuorder membar is not itself subject to
1427 // matching and translation by adlc rules. However, the rule
1428 // predicates need to detect its presence in order to correctly
1429 // select the desired adlc rules.
1430 //
1431 // Inlined unsafe volatile gets manifest as a slightly different
1432 // node sequence to a normal volatile get because of the
1433 // introduction of some CPUOrder memory barriers to bracket the
1434 // Load. However, but the same basic skeleton of a LoadX feeding a
1435 // MemBarAcquire, possibly through an optional DecodeN, is still
1436 // present
1437 //
1438 // MemBarCPUOrder
1439 // || \\
1440 // MemBarCPUOrder LoadX[mo_acquire]
1441 // || |
1442 // || {DecodeN} optional
1443 // || /
1444 // MemBarAcquire
1445 //
1446 // In this case the acquire membar does not directly depend on the
1447 // load. However, we can be sure that the load is generated from an
1448 // inlined unsafe volatile get if we see it dependent on this unique
1449 // sequence of membar nodes. Similarly, given an acquire membar we
1450 // can know that it was added because of an inlined unsafe volatile
1451 // get if it is fed and feeds a cpuorder membar and if its feed
1452 // membar also feeds an acquiring load.
1453 //
1454 // Finally an inlined (Unsafe) CAS operation is translated to the
1455 // following ideal graph
1456 //
1457 // MemBarRelease
1458 // MemBarCPUOrder
1459 // CompareAndSwapX {CardMark}-optional
1460 // MemBarCPUOrder
1461 // MemBarAcquire
1462 //
1463 // So, where we can identify these volatile read and write
1464 // signatures we can choose to plant either of the above two code
1465 // sequences. For a volatile read we can simply plant a normal
1466 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1467 // also choose to inhibit translation of the MemBarAcquire and
1468 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1469 //
1470 // When we recognise a volatile store signature we can choose to
1471 // plant at a dmb ish as a translation for the MemBarRelease, a
1472 // normal str<x> and then a dmb ish for the MemBarVolatile.
1473 // Alternatively, we can inhibit translation of the MemBarRelease
1474 // and MemBarVolatile and instead plant a simple stlr<x>
1475 // instruction.
1476 //
1477 // when we recognise a CAS signature we can choose to plant a dmb
1478 // ish as a translation for the MemBarRelease, the conventional
1479 // macro-instruction sequence for the CompareAndSwap node (which
1480 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1481 // Alternatively, we can elide generation of the dmb instructions
1482 // and plant the alternative CompareAndSwap macro-instruction
1483 // sequence (which uses ldaxr<x>).
1484 //
1485 // Of course, the above only applies when we see these signature
1486 // configurations. We still want to plant dmb instructions in any
1487 // other cases where we may see a MemBarAcquire, MemBarRelease or
1488 // MemBarVolatile. For example, at the end of a constructor which
1489 // writes final/volatile fields we will see a MemBarRelease
1490 // instruction and this needs a 'dmb ish' lest we risk the
1491 // constructed object being visible without making the
1492 // final/volatile field writes visible.
1493 //
1494 // n.b. the translation rules below which rely on detection of the
1495 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1496 // If we see anything other than the signature configurations we
1497 // always just translate the loads and stores to ldr<x> and str<x>
1498 // and translate acquire, release and volatile membars to the
1499 // relevant dmb instructions.
1500 //
1501
1502 // is_CAS(int opcode, bool maybe_volatile)
1503 //
1504 // return true if opcode is one of the possible CompareAndSwapX
1505 // values otherwise false.
1506
1507 bool is_CAS(int opcode, bool maybe_volatile)
1508 {
1509 switch(opcode) {
1510 // We handle these
1511 case Op_CompareAndSwapI:
1512 case Op_CompareAndSwapL:
1513 case Op_CompareAndSwapP:
1514 case Op_CompareAndSwapN:
1515 case Op_CompareAndSwapB:
1516 case Op_CompareAndSwapS:
1517 case Op_GetAndSetI:
1518 case Op_GetAndSetL:
1519 case Op_GetAndSetP:
1520 case Op_GetAndSetN:
1521 case Op_GetAndAddI:
1522 case Op_GetAndAddL:
1523 return true;
1524 case Op_CompareAndExchangeI:
1525 case Op_CompareAndExchangeN:
1526 case Op_CompareAndExchangeB:
1527 case Op_CompareAndExchangeS:
1528 case Op_CompareAndExchangeL:
1529 case Op_CompareAndExchangeP:
1530 case Op_WeakCompareAndSwapB:
1531 case Op_WeakCompareAndSwapS:
1532 case Op_WeakCompareAndSwapI:
1533 case Op_WeakCompareAndSwapL:
1534 case Op_WeakCompareAndSwapP:
1535 case Op_WeakCompareAndSwapN:
1536 return maybe_volatile;
1537 default:
1538 return false;
1539 }
1540 }
1541
1542 // helper to determine the maximum number of Phi nodes we may need to
1543 // traverse when searching from a card mark membar for the merge mem
1544 // feeding a trailing membar or vice versa
1545
1546 // predicates controlling emit of ldr<x>/ldar<x>
1547
1548 bool unnecessary_acquire(const Node *barrier)
1549 {
1550 assert(barrier->is_MemBar(), "expecting a membar");
1551
1552 MemBarNode* mb = barrier->as_MemBar();
1553
1554 if (mb->trailing_load()) {
1555 return true;
1556 }
1557
1558 if (mb->trailing_load_store()) {
1559 Node* load_store = mb->in(MemBarNode::Precedent);
1560 assert(load_store->is_LoadStore(), "unexpected graph shape");
1561 return is_CAS(load_store->Opcode(), true);
1562 }
1563
1564 return false;
1565 }
1566
1567 bool needs_acquiring_load(const Node *n)
1568 {
1569 assert(n->is_Load(), "expecting a load");
1570 LoadNode *ld = n->as_Load();
1571 return ld->is_acquire();
1572 }
1573
1574 bool unnecessary_release(const Node *n)
1575 {
1576 assert((n->is_MemBar() &&
1577 n->Opcode() == Op_MemBarRelease),
1578 "expecting a release membar");
1579
1580 MemBarNode *barrier = n->as_MemBar();
1581 if (!barrier->leading()) {
1582 return false;
1583 } else {
1584 Node* trailing = barrier->trailing_membar();
1585 MemBarNode* trailing_mb = trailing->as_MemBar();
1586 assert(trailing_mb->trailing(), "Not a trailing membar?");
1587 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1588
1589 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1590 if (mem->is_Store()) {
1591 assert(mem->as_Store()->is_release(), "");
1592 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1593 return true;
1594 } else {
1595 assert(mem->is_LoadStore(), "");
1596 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1597 return is_CAS(mem->Opcode(), true);
1598 }
1599 }
1600 return false;
1601 }
1602
1603 bool unnecessary_volatile(const Node *n)
1604 {
1605 // assert n->is_MemBar();
1606 MemBarNode *mbvol = n->as_MemBar();
1607
1608 bool release = mbvol->trailing_store();
1609 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1610 #ifdef ASSERT
1611 if (release) {
1612 Node* leading = mbvol->leading_membar();
1613 assert(leading->Opcode() == Op_MemBarRelease, "");
1614 assert(leading->as_MemBar()->leading_store(), "");
1615 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1616 }
1617 #endif
1618
1619 return release;
1620 }
1621
1622 // predicates controlling emit of str<x>/stlr<x>
1623
1624 bool needs_releasing_store(const Node *n)
1625 {
1626 // assert n->is_Store();
1627 StoreNode *st = n->as_Store();
1628 return st->trailing_membar() != nullptr;
1629 }
1630
1631 // predicate controlling translation of CAS
1632 //
1633 // returns true if CAS needs to use an acquiring load otherwise false
1634
1635 bool needs_acquiring_load_exclusive(const Node *n)
1636 {
1637 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1638 LoadStoreNode* ldst = n->as_LoadStore();
1639 if (is_CAS(n->Opcode(), false)) {
1640 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1641 } else {
1642 return ldst->trailing_membar() != nullptr;
1643 }
1644
1645 // so we can just return true here
1646 return true;
1647 }
1648
1649 #define __ masm->
1650
1651 // advance declarations for helper functions to convert register
1652 // indices to register objects
1653
1654 // the ad file has to provide implementations of certain methods
1655 // expected by the generic code
1656 //
1657 // REQUIRED FUNCTIONALITY
1658
1659 //=============================================================================
1660
1661 // !!!!! Special hack to get all types of calls to specify the byte offset
1662 // from the start of the call to the point where the return address
1663 // will point.
1664
1665 int MachCallStaticJavaNode::ret_addr_offset()
1666 {
1667 // call should be a simple bl
1668 int off = 4;
1669 return off;
1670 }
1671
1672 int MachCallDynamicJavaNode::ret_addr_offset()
1673 {
1674 return 16; // movz, movk, movk, bl
1675 }
1676
1677 int MachCallRuntimeNode::ret_addr_offset() {
1678 // for generated stubs the call will be
1679 // bl(addr)
1680 // or with far branches
1681 // bl(trampoline_stub)
1682 // for real runtime callouts it will be six instructions
1683 // see aarch64_enc_java_to_runtime
1684 // adr(rscratch2, retaddr)
1685 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1686 // lea(rscratch1, RuntimeAddress(addr)
1687 // blr(rscratch1)
1688 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1689 if (cb) {
1690 return 1 * NativeInstruction::instruction_size;
1691 } else if (_entry_point == nullptr) {
1692 // See CallLeafNoFPIndirect
1693 return 1 * NativeInstruction::instruction_size;
1694 } else {
1695 return 6 * NativeInstruction::instruction_size;
1696 }
1697 }
1698
1699 //=============================================================================
1700
1701 #ifndef PRODUCT
1702 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1703 st->print("BREAKPOINT");
1704 }
1705 #endif
1706
1707 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1708 __ brk(0);
1709 }
1710
1711 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1712 return MachNode::size(ra_);
1713 }
1714
1715 //=============================================================================
1716
1717 #ifndef PRODUCT
1718 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1719 st->print("nop \t# %d bytes pad for loops and calls", _count);
1720 }
1721 #endif
1722
1723 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1724 for (int i = 0; i < _count; i++) {
1725 __ nop();
1726 }
1727 }
1728
1729 uint MachNopNode::size(PhaseRegAlloc*) const {
1730 return _count * NativeInstruction::instruction_size;
1731 }
1732
1733 //=============================================================================
1734 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1735
1736 int ConstantTable::calculate_table_base_offset() const {
1737 return 0; // absolute addressing, no offset
1738 }
1739
1740 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1741 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1742 ShouldNotReachHere();
1743 }
1744
1745 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1746 // Empty encoding
1747 }
1748
1749 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1750 return 0;
1751 }
1752
1753 #ifndef PRODUCT
1754 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1755 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1756 }
1757 #endif
1758
1759 #ifndef PRODUCT
1760 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1761 Compile* C = ra_->C;
1762
1763 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1764
1765 if (C->output()->need_stack_bang(framesize))
1766 st->print("# stack bang size=%d\n\t", framesize);
1767
1768 if (VM_Version::use_rop_protection()) {
1769 st->print("ldr zr, [lr]\n\t");
1770 st->print("paciaz\n\t");
1771 }
1772 if (framesize < ((1 << 9) + 2 * wordSize)) {
1773 st->print("sub sp, sp, #%d\n\t", framesize);
1774 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1775 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1776 } else {
1777 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1778 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1779 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1780 st->print("sub sp, sp, rscratch1");
1781 }
1782 if (C->stub_function() == nullptr) {
1783 st->print("\n\t");
1784 st->print("ldr rscratch1, [guard]\n\t");
1785 st->print("dmb ishld\n\t");
1786 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1787 st->print("cmp rscratch1, rscratch2\n\t");
1788 st->print("b.eq skip");
1789 st->print("\n\t");
1790 st->print("blr #nmethod_entry_barrier_stub\n\t");
1791 st->print("b skip\n\t");
1792 st->print("guard: int\n\t");
1793 st->print("\n\t");
1794 st->print("skip:\n\t");
1795 }
1796 }
1797 #endif
1798
1799 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1800 Compile* C = ra_->C;
1801
1802
1803 __ verified_entry(C, 0);
1804
1805 if (C->stub_function() == nullptr) {
1806 __ entry_barrier();
1807 }
1808
1809 if (!Compile::current()->output()->in_scratch_emit_size()) {
1810 __ bind(*_verified_entry);
1811 }
1812
1813 if (VerifyStackAtCalls) {
1814 Unimplemented();
1815 }
1816
1817 C->output()->set_frame_complete(__ offset());
1818
1819 if (C->has_mach_constant_base_node()) {
1820 // NOTE: We set the table base offset here because users might be
1821 // emitted before MachConstantBaseNode.
1822 ConstantTable& constant_table = C->output()->constant_table();
1823 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1824 }
1825 }
1826
1827 int MachPrologNode::reloc() const
1828 {
1829 return 0;
1830 }
1831
1832 //=============================================================================
1833
1834 #ifndef PRODUCT
1835 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1836 Compile* C = ra_->C;
1837 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1838
1839 st->print("# pop frame %d\n\t",framesize);
1840
1841 if (framesize == 0) {
1842 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1843 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1844 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1845 st->print("add sp, sp, #%d\n\t", framesize);
1846 } else {
1847 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1848 st->print("add sp, sp, rscratch1\n\t");
1849 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1850 }
1851 if (VM_Version::use_rop_protection()) {
1852 st->print("autiaz\n\t");
1853 st->print("ldr zr, [lr]\n\t");
1854 }
1855
1856 if (do_polling() && C->is_method_compilation()) {
1857 st->print("# test polling word\n\t");
1858 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1859 st->print("cmp sp, rscratch1\n\t");
1860 st->print("bhi #slow_path");
1861 }
1862 }
1863 #endif
1864
1865 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1866 Compile* C = ra_->C;
1867 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1868
1869 __ remove_frame(framesize, C->needs_stack_repair());
1870
1871 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1872 __ reserved_stack_check();
1873 }
1874
1875 if (do_polling() && C->is_method_compilation()) {
1876 Label dummy_label;
1877 Label* code_stub = &dummy_label;
1878 if (!C->output()->in_scratch_emit_size()) {
1879 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1880 C->output()->add_stub(stub);
1881 code_stub = &stub->entry();
1882 }
1883 __ relocate(relocInfo::poll_return_type);
1884 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1885 }
1886 }
1887
1888 int MachEpilogNode::reloc() const {
1889 // Return number of relocatable values contained in this instruction.
1890 return 1; // 1 for polling page.
1891 }
1892
1893 const Pipeline * MachEpilogNode::pipeline() const {
1894 return MachNode::pipeline_class();
1895 }
1896
1897 //=============================================================================
1898
1899 static enum RC rc_class(OptoReg::Name reg) {
1900
1901 if (reg == OptoReg::Bad) {
1902 return rc_bad;
1903 }
1904
1905 // we have 32 int registers * 2 halves
1906 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1907
1908 if (reg < slots_of_int_registers) {
1909 return rc_int;
1910 }
1911
1912 // we have 32 float register * 8 halves
1913 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1914 if (reg < slots_of_int_registers + slots_of_float_registers) {
1915 return rc_float;
1916 }
1917
1918 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1919 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1920 return rc_predicate;
1921 }
1922
1923 // Between predicate regs & stack is the flags.
1924 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1925
1926 return rc_stack;
1927 }
1928
1929 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1930 Compile* C = ra_->C;
1931
1932 // Get registers to move.
1933 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1934 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1935 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1936 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1937
1938 enum RC src_hi_rc = rc_class(src_hi);
1939 enum RC src_lo_rc = rc_class(src_lo);
1940 enum RC dst_hi_rc = rc_class(dst_hi);
1941 enum RC dst_lo_rc = rc_class(dst_lo);
1942
1943 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1944
1945 if (src_hi != OptoReg::Bad && !bottom_type()->isa_pvectmask()) {
1946 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1947 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1948 "expected aligned-adjacent pairs");
1949 }
1950
1951 if (src_lo == dst_lo && src_hi == dst_hi) {
1952 return 0; // Self copy, no move.
1953 }
1954
1955 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1956 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1957 int src_offset = ra_->reg2offset(src_lo);
1958 int dst_offset = ra_->reg2offset(dst_lo);
1959
1960 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
1961 uint ireg = ideal_reg();
1962 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
1963 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
1964 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
1965 if (ireg == Op_VecA && masm) {
1966 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1967 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1968 // stack->stack
1969 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1970 sve_vector_reg_size_in_bytes);
1971 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1972 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1973 sve_vector_reg_size_in_bytes);
1974 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1975 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1976 sve_vector_reg_size_in_bytes);
1977 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1978 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1979 as_FloatRegister(Matcher::_regEncode[src_lo]),
1980 as_FloatRegister(Matcher::_regEncode[src_lo]));
1981 } else {
1982 ShouldNotReachHere();
1983 }
1984 } else if (masm) {
1985 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1986 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1987 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1988 // stack->stack
1989 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1990 if (ireg == Op_VecD) {
1991 __ unspill(rscratch1, true, src_offset);
1992 __ spill(rscratch1, true, dst_offset);
1993 } else {
1994 __ spill_copy128(src_offset, dst_offset);
1995 }
1996 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1997 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1998 ireg == Op_VecD ? __ T8B : __ T16B,
1999 as_FloatRegister(Matcher::_regEncode[src_lo]));
2000 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2001 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2002 ireg == Op_VecD ? __ D : __ Q,
2003 ra_->reg2offset(dst_lo));
2004 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2005 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2006 ireg == Op_VecD ? __ D : __ Q,
2007 ra_->reg2offset(src_lo));
2008 } else {
2009 ShouldNotReachHere();
2010 }
2011 }
2012 } else if (masm) {
2013 switch (src_lo_rc) {
2014 case rc_int:
2015 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2016 if (is64) {
2017 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2018 as_Register(Matcher::_regEncode[src_lo]));
2019 } else {
2020 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2021 as_Register(Matcher::_regEncode[src_lo]));
2022 }
2023 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2024 if (is64) {
2025 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2026 as_Register(Matcher::_regEncode[src_lo]));
2027 } else {
2028 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2029 as_Register(Matcher::_regEncode[src_lo]));
2030 }
2031 } else { // gpr --> stack spill
2032 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2033 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2034 }
2035 break;
2036 case rc_float:
2037 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2038 if (is64) {
2039 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2040 as_FloatRegister(Matcher::_regEncode[src_lo]));
2041 } else {
2042 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2043 as_FloatRegister(Matcher::_regEncode[src_lo]));
2044 }
2045 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2046 if (is64) {
2047 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2048 as_FloatRegister(Matcher::_regEncode[src_lo]));
2049 } else {
2050 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2051 as_FloatRegister(Matcher::_regEncode[src_lo]));
2052 }
2053 } else { // fpr --> stack spill
2054 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2055 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2056 is64 ? __ D : __ S, dst_offset);
2057 }
2058 break;
2059 case rc_stack:
2060 if (dst_lo_rc == rc_int) { // stack --> gpr load
2061 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2062 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2063 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2064 is64 ? __ D : __ S, src_offset);
2065 } else if (dst_lo_rc == rc_predicate) {
2066 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2067 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2068 } else { // stack --> stack copy
2069 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2070 if (ideal_reg() == Op_RegVectMask) {
2071 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2072 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2073 } else {
2074 __ unspill(rscratch1, is64, src_offset);
2075 __ spill(rscratch1, is64, dst_offset);
2076 }
2077 }
2078 break;
2079 case rc_predicate:
2080 if (dst_lo_rc == rc_predicate) {
2081 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2082 } else if (dst_lo_rc == rc_stack) {
2083 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2084 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2085 } else {
2086 assert(false, "bad src and dst rc_class combination.");
2087 ShouldNotReachHere();
2088 }
2089 break;
2090 default:
2091 assert(false, "bad rc_class for spill");
2092 ShouldNotReachHere();
2093 }
2094 }
2095
2096 if (st) {
2097 st->print("spill ");
2098 if (src_lo_rc == rc_stack) {
2099 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2100 } else {
2101 st->print("%s -> ", Matcher::regName[src_lo]);
2102 }
2103 if (dst_lo_rc == rc_stack) {
2104 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2105 } else {
2106 st->print("%s", Matcher::regName[dst_lo]);
2107 }
2108 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2109 int vsize = 0;
2110 switch (ideal_reg()) {
2111 case Op_VecD:
2112 vsize = 64;
2113 break;
2114 case Op_VecX:
2115 vsize = 128;
2116 break;
2117 case Op_VecA:
2118 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2119 break;
2120 default:
2121 assert(false, "bad register type for spill");
2122 ShouldNotReachHere();
2123 }
2124 st->print("\t# vector spill size = %d", vsize);
2125 } else if (ideal_reg() == Op_RegVectMask) {
2126 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2127 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2128 st->print("\t# predicate spill size = %d", vsize);
2129 } else {
2130 st->print("\t# spill size = %d", is64 ? 64 : 32);
2131 }
2132 }
2133
2134 return 0;
2135
2136 }
2137
2138 #ifndef PRODUCT
2139 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2140 if (!ra_)
2141 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2142 else
2143 implementation(nullptr, ra_, false, st);
2144 }
2145 #endif
2146
2147 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2148 implementation(masm, ra_, false, nullptr);
2149 }
2150
2151 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2152 return MachNode::size(ra_);
2153 }
2154
2155 //=============================================================================
2156
2157 #ifndef PRODUCT
2158 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2159 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2160 int reg = ra_->get_reg_first(this);
2161 st->print("add %s, rsp, #%d]\t# box lock",
2162 Matcher::regName[reg], offset);
2163 }
2164 #endif
2165
2166 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2167 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2168 int reg = ra_->get_encode(this);
2169
2170 // This add will handle any 24-bit signed offset. 24 bits allows an
2171 // 8 megabyte stack frame.
2172 __ add(as_Register(reg), sp, offset);
2173 }
2174
2175 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2176 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2177 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2178
2179 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2180 return NativeInstruction::instruction_size;
2181 } else {
2182 return 2 * NativeInstruction::instruction_size;
2183 }
2184 }
2185
2186 ///=============================================================================
2187 #ifndef PRODUCT
2188 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2189 {
2190 st->print_cr("# MachVEPNode");
2191 if (!_verified) {
2192 st->print_cr("\t load_class");
2193 } else {
2194 st->print_cr("\t unpack_inline_arg");
2195 }
2196 }
2197 #endif
2198
2199 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2200 {
2201 if (!_verified) {
2202 __ ic_check(1);
2203 } else {
2204 if (ra_->C->stub_function() == nullptr) {
2205 // Emit the entry barrier in a temporary frame before unpacking because
2206 // it can deopt, which would require packing the scalarized args again.
2207 __ verified_entry(ra_->C, 0);
2208 __ entry_barrier();
2209 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2210 __ remove_frame(framesize, false);
2211 }
2212 // Unpack inline type args passed as oop and then jump to
2213 // the verified entry point (skipping the unverified entry).
2214 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2215 // Emit code for verified entry and save increment for stack repair on return
2216 __ verified_entry(ra_->C, sp_inc);
2217 if (Compile::current()->output()->in_scratch_emit_size()) {
2218 Label dummy_verified_entry;
2219 __ b(dummy_verified_entry);
2220 } else {
2221 __ b(*_verified_entry);
2222 }
2223 }
2224 }
2225
2226 //=============================================================================
2227 #ifndef PRODUCT
2228 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2229 {
2230 st->print_cr("# MachUEPNode");
2231 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2232 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2233 st->print_cr("\tcmpw rscratch1, r10");
2234 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2235 }
2236 #endif
2237
2238 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2239 {
2240 __ ic_check(InteriorEntryAlignment);
2241 }
2242
2243 // REQUIRED EMIT CODE
2244
2245 //=============================================================================
2246
2247 // Emit deopt handler code.
2248 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2249 {
2250 // Note that the code buffer's insts_mark is always relative to insts.
2251 // That's why we must use the macroassembler to generate a handler.
2252 address base = __ start_a_stub(size_deopt_handler());
2253 if (base == nullptr) {
2254 ciEnv::current()->record_failure("CodeCache is full");
2255 return 0; // CodeBuffer::expand failed
2256 }
2257
2258 int offset = __ offset();
2259 Label start;
2260 __ bind(start);
2261 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2262
2263 int entry_offset = __ offset();
2264 __ b(start);
2265
2266 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2267 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2268 "out of bounds read in post-call NOP check");
2269 __ end_a_stub();
2270 return entry_offset;
2271 }
2272
2273 // REQUIRED MATCHER CODE
2274
2275 //=============================================================================
2276
2277 bool Matcher::match_rule_supported(int opcode) {
2278 if (!has_match_rule(opcode))
2279 return false;
2280
2281 switch (opcode) {
2282 case Op_OnSpinWait:
2283 return VM_Version::supports_on_spin_wait();
2284 case Op_CacheWB:
2285 case Op_CacheWBPreSync:
2286 case Op_CacheWBPostSync:
2287 if (!VM_Version::supports_data_cache_line_flush()) {
2288 return false;
2289 }
2290 break;
2291 case Op_ExpandBits:
2292 case Op_CompressBits:
2293 if (!VM_Version::supports_svebitperm()) {
2294 return false;
2295 }
2296 break;
2297 case Op_FmaF:
2298 case Op_FmaD:
2299 case Op_FmaVF:
2300 case Op_FmaVD:
2301 if (!UseFMA) {
2302 return false;
2303 }
2304 break;
2305 case Op_FmaHF:
2306 // UseFMA flag also needs to be checked along with FEAT_FP16
2307 if (!UseFMA || !is_feat_fp16_supported()) {
2308 return false;
2309 }
2310 break;
2311 case Op_AddHF:
2312 case Op_SubHF:
2313 case Op_MulHF:
2314 case Op_DivHF:
2315 case Op_MinHF:
2316 case Op_MaxHF:
2317 case Op_SqrtHF:
2318 // Half-precision floating point scalar operations require FEAT_FP16
2319 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2320 // features are supported.
2321 if (!is_feat_fp16_supported()) {
2322 return false;
2323 }
2324 break;
2325 }
2326
2327 return true; // Per default match rules are supported.
2328 }
2329
2330 const RegMask* Matcher::predicate_reg_mask(void) {
2331 return &_PR_REG_mask;
2332 }
2333
2334 bool Matcher::supports_vector_calling_convention(void) {
2335 return EnableVectorSupport;
2336 }
2337
2338 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2339 assert(EnableVectorSupport, "sanity");
2340 int lo = V0_num;
2341 int hi = V0_H_num;
2342 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2343 hi = V0_K_num;
2344 }
2345 return OptoRegPair(hi, lo);
2346 }
2347
2348 // Is this branch offset short enough that a short branch can be used?
2349 //
2350 // NOTE: If the platform does not provide any short branch variants, then
2351 // this method should return false for offset 0.
2352 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2353 // The passed offset is relative to address of the branch.
2354
2355 return (-32768 <= offset && offset < 32768);
2356 }
2357
2358 // Vector width in bytes.
2359 int Matcher::vector_width_in_bytes(BasicType bt) {
2360 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2361 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2362 // Minimum 2 values in vector
2363 if (size < 2*type2aelembytes(bt)) size = 0;
2364 // But never < 4
2365 if (size < 4) size = 0;
2366 return size;
2367 }
2368
2369 // Limits on vector size (number of elements) loaded into vector.
2370 int Matcher::max_vector_size(const BasicType bt) {
2371 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2372 }
2373
2374 int Matcher::min_vector_size(const BasicType bt) {
2375 // Usually, the shortest vector length supported by AArch64 ISA and
2376 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2377 // vectors in a few special cases.
2378 int size;
2379 switch(bt) {
2380 case T_BOOLEAN:
2381 // Load/store a vector mask with only 2 elements for vector types
2382 // such as "2I/2F/2L/2D".
2383 size = 2;
2384 break;
2385 case T_BYTE:
2386 // Generate a "4B" vector, to support vector cast between "8B/16B"
2387 // and "4S/4I/4L/4F/4D".
2388 size = 4;
2389 break;
2390 case T_SHORT:
2391 // Generate a "2S" vector, to support vector cast between "4S/8S"
2392 // and "2I/2L/2F/2D".
2393 size = 2;
2394 break;
2395 default:
2396 // Limit the min vector length to 64-bit.
2397 size = 8 / type2aelembytes(bt);
2398 // The number of elements in a vector should be at least 2.
2399 size = MAX2(size, 2);
2400 }
2401
2402 int max_size = max_vector_size(bt);
2403 return MIN2(size, max_size);
2404 }
2405
2406 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2407 return Matcher::max_vector_size(bt);
2408 }
2409
2410 // Actual max scalable vector register length.
2411 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2412 return Matcher::max_vector_size(bt);
2413 }
2414
2415 // Vector ideal reg.
2416 uint Matcher::vector_ideal_reg(int len) {
2417 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2418 return Op_VecA;
2419 }
2420 switch(len) {
2421 // For 16-bit/32-bit mask vector, reuse VecD.
2422 case 2:
2423 case 4:
2424 case 8: return Op_VecD;
2425 case 16: return Op_VecX;
2426 }
2427 ShouldNotReachHere();
2428 return 0;
2429 }
2430
2431 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2432 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2433 switch (ideal_reg) {
2434 case Op_VecA: return new vecAOper();
2435 case Op_VecD: return new vecDOper();
2436 case Op_VecX: return new vecXOper();
2437 }
2438 ShouldNotReachHere();
2439 return nullptr;
2440 }
2441
2442 bool Matcher::is_reg2reg_move(MachNode* m) {
2443 return false;
2444 }
2445
2446 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2447 return false;
2448 }
2449
2450 bool Matcher::is_generic_vector(MachOper* opnd) {
2451 return opnd->opcode() == VREG;
2452 }
2453
2454 #ifdef ASSERT
2455 // Return whether or not this register is ever used as an argument.
2456 bool Matcher::can_be_java_arg(int reg)
2457 {
2458 return
2459 reg == R0_num || reg == R0_H_num ||
2460 reg == R1_num || reg == R1_H_num ||
2461 reg == R2_num || reg == R2_H_num ||
2462 reg == R3_num || reg == R3_H_num ||
2463 reg == R4_num || reg == R4_H_num ||
2464 reg == R5_num || reg == R5_H_num ||
2465 reg == R6_num || reg == R6_H_num ||
2466 reg == R7_num || reg == R7_H_num ||
2467 reg == V0_num || reg == V0_H_num ||
2468 reg == V1_num || reg == V1_H_num ||
2469 reg == V2_num || reg == V2_H_num ||
2470 reg == V3_num || reg == V3_H_num ||
2471 reg == V4_num || reg == V4_H_num ||
2472 reg == V5_num || reg == V5_H_num ||
2473 reg == V6_num || reg == V6_H_num ||
2474 reg == V7_num || reg == V7_H_num;
2475 }
2476 #endif
2477
2478 uint Matcher::int_pressure_limit()
2479 {
2480 // JDK-8183543: When taking the number of available registers as int
2481 // register pressure threshold, the jtreg test:
2482 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2483 // failed due to C2 compilation failure with
2484 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2485 //
2486 // A derived pointer is live at CallNode and then is flagged by RA
2487 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2488 // derived pointers and lastly fail to spill after reaching maximum
2489 // number of iterations. Lowering the default pressure threshold to
2490 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2491 // a high register pressure area of the code so that split_DEF can
2492 // generate DefinitionSpillCopy for the derived pointer.
2493 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2494 if (!PreserveFramePointer) {
2495 // When PreserveFramePointer is off, frame pointer is allocatable,
2496 // but different from other SOC registers, it is excluded from
2497 // fatproj's mask because its save type is No-Save. Decrease 1 to
2498 // ensure high pressure at fatproj when PreserveFramePointer is off.
2499 // See check_pressure_at_fatproj().
2500 default_int_pressure_threshold--;
2501 }
2502 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2503 }
2504
2505 uint Matcher::float_pressure_limit()
2506 {
2507 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2508 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2509 }
2510
2511 const RegMask& Matcher::firstI_proj_mask() {
2512 ShouldNotReachHere();
2513 return RegMask::EMPTY;
2514 }
2515
2516 // Register for the second projection of an int pair
2517 const RegMask& Matcher::secondI_proj_mask() {
2518 ShouldNotReachHere();
2519 return RegMask::EMPTY;
2520 }
2521
2522 // Register for the first projection of a long pair
2523 const RegMask& Matcher::firstL_proj_mask() {
2524 ShouldNotReachHere();
2525 return RegMask::EMPTY;
2526 }
2527
2528 // Register for the second projection of a long pair
2529 const RegMask& Matcher::secondL_proj_mask() {
2530 ShouldNotReachHere();
2531 return RegMask::EMPTY;
2532 }
2533
2534 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2535 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2536 Node* u = addp->fast_out(i);
2537 if (u->is_LoadStore()) {
2538 // On AArch64, LoadStoreNodes (i.e. compare and swap
2539 // instructions) only take register indirect as an operand, so
2540 // any attempt to use an AddPNode as an input to a LoadStoreNode
2541 // must fail.
2542 return false;
2543 }
2544 if (u->is_Mem()) {
2545 int opsize = u->as_Mem()->memory_size();
2546 assert(opsize > 0, "unexpected memory operand size");
2547 if (u->as_Mem()->memory_size() != (1<<shift)) {
2548 return false;
2549 }
2550 }
2551 }
2552 return true;
2553 }
2554
2555 // Convert BoolTest condition to Assembler condition.
2556 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2557 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2558 Assembler::Condition result;
2559 switch(cond) {
2560 case BoolTest::eq:
2561 result = Assembler::EQ; break;
2562 case BoolTest::ne:
2563 result = Assembler::NE; break;
2564 case BoolTest::le:
2565 result = Assembler::LE; break;
2566 case BoolTest::ge:
2567 result = Assembler::GE; break;
2568 case BoolTest::lt:
2569 result = Assembler::LT; break;
2570 case BoolTest::gt:
2571 result = Assembler::GT; break;
2572 case BoolTest::ule:
2573 result = Assembler::LS; break;
2574 case BoolTest::uge:
2575 result = Assembler::HS; break;
2576 case BoolTest::ult:
2577 result = Assembler::LO; break;
2578 case BoolTest::ugt:
2579 result = Assembler::HI; break;
2580 case BoolTest::overflow:
2581 result = Assembler::VS; break;
2582 case BoolTest::no_overflow:
2583 result = Assembler::VC; break;
2584 default:
2585 ShouldNotReachHere();
2586 return Assembler::Condition(-1);
2587 }
2588
2589 // Check conversion
2590 if (cond & BoolTest::unsigned_compare) {
2591 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2592 } else {
2593 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2594 }
2595
2596 return result;
2597 }
2598
2599 // Binary src (Replicate con)
2600 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2601 if (n == nullptr || m == nullptr) {
2602 return false;
2603 }
2604
2605 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2606 return false;
2607 }
2608
2609 Node* imm_node = m->in(1);
2610 if (!imm_node->is_Con()) {
2611 return false;
2612 }
2613
2614 const Type* t = imm_node->bottom_type();
2615 if (!(t->isa_int() || t->isa_long())) {
2616 return false;
2617 }
2618
2619 switch (n->Opcode()) {
2620 case Op_AndV:
2621 case Op_OrV:
2622 case Op_XorV: {
2623 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2624 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2625 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2626 }
2627 case Op_AddVB:
2628 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2629 case Op_AddVS:
2630 case Op_AddVI:
2631 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2632 case Op_AddVL:
2633 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2634 default:
2635 return false;
2636 }
2637 }
2638
2639 // (XorV src (Replicate m1))
2640 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2641 if (n != nullptr && m != nullptr) {
2642 return n->Opcode() == Op_XorV &&
2643 VectorNode::is_all_ones_vector(m);
2644 }
2645 return false;
2646 }
2647
2648 // Returns true if (n, m) matches "(XorVMask vm2 (MaskAll m1))" and that XorVMask
2649 // is used only by an AndVMask. In that case, cloning m (the MaskAll) lets the
2650 // matcher avoid sharing the MaskAll node and subsume the pattern into rule:
2651 // "(AndVMask vm1 (XorVMask vm2 (MaskAll m1)))".
2652 //
2653 // Limitation: the "andNot" rule still cannot be matched if "m" has other
2654 // uses outside this pattern.
2655 static bool is_vector_mask_not_operand_in_andnot_pattern(Node* n, Node* m) {
2656 if (n == nullptr || m == nullptr) {
2657 return false;
2658 }
2659
2660 if (VectorNode::is_all_ones_vector(m) &&
2661 n->Opcode() == Op_XorVMask &&
2662 n->outcnt() == 1 &&
2663 n->unique_out()->Opcode() == Op_AndVMask) {
2664 // If another input of the AndVMask is also a mask-not pattern that would
2665 // qualify for the `maskAll` cloning, do not clone the "maskAll" here,
2666 // because the match rule can only consume one such pattern.
2667 Node* use = n->unique_out();
2668 Node* other_input = use->in(1) == n ? use->in(2) : use->in(1);
2669 return !VectorNode::is_vectormask_bitwise_not_pattern(other_input);
2670 }
2671 return false;
2672 }
2673
2674 // Should the matcher clone input 'm' of node 'n'?
2675 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2676 if (is_vshift_con_pattern(n, m) ||
2677 is_vector_bitwise_not_pattern(n, m) ||
2678 is_vector_mask_not_operand_in_andnot_pattern(n, m) ||
2679 is_valid_sve_arith_imm_pattern(n, m) ||
2680 is_encode_and_store_pattern(n, m)) {
2681 mstack.push(m, Visit);
2682 return true;
2683 }
2684 return false;
2685 }
2686
2687 // Should the Matcher clone shifts on addressing modes, expecting them
2688 // to be subsumed into complex addressing expressions or compute them
2689 // into registers?
2690 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2691
2692 // Loads and stores with indirect memory input (e.g., volatile loads and
2693 // stores) do not subsume the input into complex addressing expressions. If
2694 // the addressing expression is input to at least one such load or store, do
2695 // not clone the addressing expression. Query needs_acquiring_load and
2696 // needs_releasing_store as a proxy for indirect memory input, as it is not
2697 // possible to directly query for indirect memory input at this stage.
2698 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2699 Node* n = m->fast_out(i);
2700 if (n->is_Load() && needs_acquiring_load(n)) {
2701 return false;
2702 }
2703 if (n->is_Store() && needs_releasing_store(n)) {
2704 return false;
2705 }
2706 }
2707
2708 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2709 return true;
2710 }
2711
2712 Node *off = m->in(AddPNode::Offset);
2713 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2714 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2715 // Are there other uses besides address expressions?
2716 !is_visited(off)) {
2717 address_visited.set(off->_idx); // Flag as address_visited
2718 mstack.push(off->in(2), Visit);
2719 Node *conv = off->in(1);
2720 if (conv->Opcode() == Op_ConvI2L &&
2721 // Are there other uses besides address expressions?
2722 !is_visited(conv)) {
2723 address_visited.set(conv->_idx); // Flag as address_visited
2724 mstack.push(conv->in(1), Pre_Visit);
2725 } else {
2726 mstack.push(conv, Pre_Visit);
2727 }
2728 address_visited.test_set(m->_idx); // Flag as address_visited
2729 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2730 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2731 return true;
2732 } else if (off->Opcode() == Op_ConvI2L &&
2733 // Are there other uses besides address expressions?
2734 !is_visited(off)) {
2735 address_visited.test_set(m->_idx); // Flag as address_visited
2736 address_visited.set(off->_idx); // Flag as address_visited
2737 mstack.push(off->in(1), Pre_Visit);
2738 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2739 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2740 return true;
2741 }
2742 return false;
2743 }
2744
2745 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2746 { \
2747 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2748 guarantee(DISP == 0, "mode not permitted for volatile"); \
2749 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2750 __ INSN(REG, as_Register(BASE)); \
2751 }
2752
2753
2754 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2755 {
2756 Address::extend scale;
2757
2758 // Hooboy, this is fugly. We need a way to communicate to the
2759 // encoder that the index needs to be sign extended, so we have to
2760 // enumerate all the cases.
2761 switch (opcode) {
2762 case INDINDEXSCALEDI2L:
2763 case INDINDEXSCALEDI2LN:
2764 case INDINDEXI2L:
2765 case INDINDEXI2LN:
2766 scale = Address::sxtw(size);
2767 break;
2768 default:
2769 scale = Address::lsl(size);
2770 }
2771
2772 if (index == -1) {
2773 return Address(base, disp);
2774 } else {
2775 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2776 return Address(base, as_Register(index), scale);
2777 }
2778 }
2779
2780
2781 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2782 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2783 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2784 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2785 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2786
2787 // Used for all non-volatile memory accesses. The use of
2788 // $mem->opcode() to discover whether this pattern uses sign-extended
2789 // offsets is something of a kludge.
2790 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2791 Register reg, int opcode,
2792 Register base, int index, int scale, int disp,
2793 int size_in_memory)
2794 {
2795 Address addr = mem2address(opcode, base, index, scale, disp);
2796 if (addr.getMode() == Address::base_plus_offset) {
2797 /* Fix up any out-of-range offsets. */
2798 assert_different_registers(rscratch1, base);
2799 assert_different_registers(rscratch1, reg);
2800 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2801 }
2802 (masm->*insn)(reg, addr);
2803 }
2804
2805 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2806 FloatRegister reg, int opcode,
2807 Register base, int index, int size, int disp,
2808 int size_in_memory)
2809 {
2810 Address::extend scale;
2811
2812 switch (opcode) {
2813 case INDINDEXSCALEDI2L:
2814 case INDINDEXSCALEDI2LN:
2815 scale = Address::sxtw(size);
2816 break;
2817 default:
2818 scale = Address::lsl(size);
2819 }
2820
2821 if (index == -1) {
2822 // Fix up any out-of-range offsets.
2823 assert_different_registers(rscratch1, base);
2824 Address addr = Address(base, disp);
2825 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2826 (masm->*insn)(reg, addr);
2827 } else {
2828 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2829 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2830 }
2831 }
2832
2833 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2834 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2835 int opcode, Register base, int index, int size, int disp)
2836 {
2837 if (index == -1) {
2838 (masm->*insn)(reg, T, Address(base, disp));
2839 } else {
2840 assert(disp == 0, "unsupported address mode");
2841 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2842 }
2843 }
2844
2845 %}
2846
2847
2848
2849 //----------ENCODING BLOCK-----------------------------------------------------
2850 // This block specifies the encoding classes used by the compiler to
2851 // output byte streams. Encoding classes are parameterized macros
2852 // used by Machine Instruction Nodes in order to generate the bit
2853 // encoding of the instruction. Operands specify their base encoding
2854 // interface with the interface keyword. There are currently
2855 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2856 // COND_INTER. REG_INTER causes an operand to generate a function
2857 // which returns its register number when queried. CONST_INTER causes
2858 // an operand to generate a function which returns the value of the
2859 // constant when queried. MEMORY_INTER causes an operand to generate
2860 // four functions which return the Base Register, the Index Register,
2861 // the Scale Value, and the Offset Value of the operand when queried.
2862 // COND_INTER causes an operand to generate six functions which return
2863 // the encoding code (ie - encoding bits for the instruction)
2864 // associated with each basic boolean condition for a conditional
2865 // instruction.
2866 //
2867 // Instructions specify two basic values for encoding. Again, a
2868 // function is available to check if the constant displacement is an
2869 // oop. They use the ins_encode keyword to specify their encoding
2870 // classes (which must be a sequence of enc_class names, and their
2871 // parameters, specified in the encoding block), and they use the
2872 // opcode keyword to specify, in order, their primary, secondary, and
2873 // tertiary opcode. Only the opcode sections which a particular
2874 // instruction needs for encoding need to be specified.
2875 encode %{
2876 // Build emit functions for each basic byte or larger field in the
2877 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2878 // from C++ code in the enc_class source block. Emit functions will
2879 // live in the main source block for now. In future, we can
2880 // generalize this by adding a syntax that specifies the sizes of
2881 // fields in an order, so that the adlc can build the emit functions
2882 // automagically
2883
2884 // catch all for unimplemented encodings
2885 enc_class enc_unimplemented %{
2886 __ unimplemented("C2 catch all");
2887 %}
2888
2889 // BEGIN Non-volatile memory access
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2934 Register dst_reg = as_Register($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2942 Register dst_reg = as_Register($dst$$reg);
2943 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2950 Register dst_reg = as_Register($dst$$reg);
2951 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2958 Register dst_reg = as_Register($dst$$reg);
2959 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2966 Register dst_reg = as_Register($dst$$reg);
2967 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2974 Register dst_reg = as_Register($dst$$reg);
2975 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2976 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2977 %}
2978
2979 // This encoding class is generated automatically from ad_encode.m4.
2980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2981 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2982 Register dst_reg = as_Register($dst$$reg);
2983 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2984 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2985 %}
2986
2987 // This encoding class is generated automatically from ad_encode.m4.
2988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2989 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2990 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2991 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2998 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2999 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3000 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3001 %}
3002
3003 // This encoding class is generated automatically from ad_encode.m4.
3004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3005 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3006 Register src_reg = as_Register($src$$reg);
3007 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3008 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3009 %}
3010
3011 // This encoding class is generated automatically from ad_encode.m4.
3012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3013 enc_class aarch64_enc_strb0(memory1 mem) %{
3014 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3015 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3016 %}
3017
3018 // This encoding class is generated automatically from ad_encode.m4.
3019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3020 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3021 Register src_reg = as_Register($src$$reg);
3022 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_strh0(memory2 mem) %{
3029 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3030 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3031 %}
3032
3033 // This encoding class is generated automatically from ad_encode.m4.
3034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3035 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3036 Register src_reg = as_Register($src$$reg);
3037 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3038 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3039 %}
3040
3041 // This encoding class is generated automatically from ad_encode.m4.
3042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3043 enc_class aarch64_enc_strw0(memory4 mem) %{
3044 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3045 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3046 %}
3047
3048 // This encoding class is generated automatically from ad_encode.m4.
3049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3050 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3051 Register src_reg = as_Register($src$$reg);
3052 // we sometimes get asked to store the stack pointer into the
3053 // current thread -- we cannot do that directly on AArch64
3054 if (src_reg == r31_sp) {
3055 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3056 __ mov(rscratch2, sp);
3057 src_reg = rscratch2;
3058 }
3059 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3060 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3061 %}
3062
3063 // This encoding class is generated automatically from ad_encode.m4.
3064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3065 enc_class aarch64_enc_str0(memory8 mem) %{
3066 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3067 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3068 %}
3069
3070 // This encoding class is generated automatically from ad_encode.m4.
3071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3072 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3073 FloatRegister src_reg = as_FloatRegister($src$$reg);
3074 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3075 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3076 %}
3077
3078 // This encoding class is generated automatically from ad_encode.m4.
3079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3080 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3081 FloatRegister src_reg = as_FloatRegister($src$$reg);
3082 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3083 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3084 %}
3085
3086 // This encoding class is generated automatically from ad_encode.m4.
3087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3088 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3089 __ membar(Assembler::StoreStore);
3090 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3091 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3092 %}
3093
3094 // END Non-volatile memory access
3095
3096 // Vector loads and stores
3097 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3098 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3099 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3100 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3101 %}
3102
3103 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3104 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3105 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3106 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3107 %}
3108
3109 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3110 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3111 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3112 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3113 %}
3114
3115 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3116 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3117 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3118 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3119 %}
3120
3121 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3122 FloatRegister src_reg = as_FloatRegister($src$$reg);
3123 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3124 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3125 %}
3126
3127 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3128 FloatRegister src_reg = as_FloatRegister($src$$reg);
3129 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3130 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3131 %}
3132
3133 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3134 FloatRegister src_reg = as_FloatRegister($src$$reg);
3135 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3136 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3137 %}
3138
3139 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3140 FloatRegister src_reg = as_FloatRegister($src$$reg);
3141 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3142 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3143 %}
3144
3145 // volatile loads and stores
3146
3147 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3148 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, stlrb);
3150 %}
3151
3152 enc_class aarch64_enc_stlrb0(memory mem) %{
3153 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, stlrb);
3155 %}
3156
3157 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3158 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3159 rscratch1, stlrh);
3160 %}
3161
3162 enc_class aarch64_enc_stlrh0(memory mem) %{
3163 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, stlrh);
3165 %}
3166
3167 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3168 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, stlrw);
3170 %}
3171
3172 enc_class aarch64_enc_stlrw0(memory mem) %{
3173 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3174 rscratch1, stlrw);
3175 %}
3176
3177 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3178 Register dst_reg = as_Register($dst$$reg);
3179 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3180 rscratch1, ldarb);
3181 __ sxtbw(dst_reg, dst_reg);
3182 %}
3183
3184 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3185 Register dst_reg = as_Register($dst$$reg);
3186 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3187 rscratch1, ldarb);
3188 __ sxtb(dst_reg, dst_reg);
3189 %}
3190
3191 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3192 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3193 rscratch1, ldarb);
3194 %}
3195
3196 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3197 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, ldarb);
3199 %}
3200
3201 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3202 Register dst_reg = as_Register($dst$$reg);
3203 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3204 rscratch1, ldarh);
3205 __ sxthw(dst_reg, dst_reg);
3206 %}
3207
3208 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3209 Register dst_reg = as_Register($dst$$reg);
3210 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3211 rscratch1, ldarh);
3212 __ sxth(dst_reg, dst_reg);
3213 %}
3214
3215 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3216 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3217 rscratch1, ldarh);
3218 %}
3219
3220 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3221 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3222 rscratch1, ldarh);
3223 %}
3224
3225 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3226 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3227 rscratch1, ldarw);
3228 %}
3229
3230 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3231 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, ldarw);
3233 %}
3234
3235 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3236 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3237 rscratch1, ldar);
3238 %}
3239
3240 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3241 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3242 rscratch1, ldarw);
3243 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3244 %}
3245
3246 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3247 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3248 rscratch1, ldar);
3249 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3250 %}
3251
3252 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3253 Register src_reg = as_Register($src$$reg);
3254 // we sometimes get asked to store the stack pointer into the
3255 // current thread -- we cannot do that directly on AArch64
3256 if (src_reg == r31_sp) {
3257 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3258 __ mov(rscratch2, sp);
3259 src_reg = rscratch2;
3260 }
3261 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3262 rscratch1, stlr);
3263 %}
3264
3265 enc_class aarch64_enc_stlr0(memory mem) %{
3266 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3267 rscratch1, stlr);
3268 %}
3269
3270 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3271 {
3272 FloatRegister src_reg = as_FloatRegister($src$$reg);
3273 __ fmovs(rscratch2, src_reg);
3274 }
3275 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3276 rscratch1, stlrw);
3277 %}
3278
3279 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3280 {
3281 FloatRegister src_reg = as_FloatRegister($src$$reg);
3282 __ fmovd(rscratch2, src_reg);
3283 }
3284 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3285 rscratch1, stlr);
3286 %}
3287
3288 // synchronized read/update encodings
3289
3290 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3291 Register dst_reg = as_Register($dst$$reg);
3292 Register base = as_Register($mem$$base);
3293 int index = $mem$$index;
3294 int scale = $mem$$scale;
3295 int disp = $mem$$disp;
3296 if (index == -1) {
3297 if (disp != 0) {
3298 __ lea(rscratch1, Address(base, disp));
3299 __ ldaxr(dst_reg, rscratch1);
3300 } else {
3301 // TODO
3302 // should we ever get anything other than this case?
3303 __ ldaxr(dst_reg, base);
3304 }
3305 } else {
3306 Register index_reg = as_Register(index);
3307 if (disp == 0) {
3308 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3309 __ ldaxr(dst_reg, rscratch1);
3310 } else {
3311 __ lea(rscratch1, Address(base, disp));
3312 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3313 __ ldaxr(dst_reg, rscratch1);
3314 }
3315 }
3316 %}
3317
3318 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3319 Register src_reg = as_Register($src$$reg);
3320 Register base = as_Register($mem$$base);
3321 int index = $mem$$index;
3322 int scale = $mem$$scale;
3323 int disp = $mem$$disp;
3324 if (index == -1) {
3325 if (disp != 0) {
3326 __ lea(rscratch2, Address(base, disp));
3327 __ stlxr(rscratch1, src_reg, rscratch2);
3328 } else {
3329 // TODO
3330 // should we ever get anything other than this case?
3331 __ stlxr(rscratch1, src_reg, base);
3332 }
3333 } else {
3334 Register index_reg = as_Register(index);
3335 if (disp == 0) {
3336 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3337 __ stlxr(rscratch1, src_reg, rscratch2);
3338 } else {
3339 __ lea(rscratch2, Address(base, disp));
3340 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3341 __ stlxr(rscratch1, src_reg, rscratch2);
3342 }
3343 }
3344 __ cmpw(rscratch1, zr);
3345 %}
3346
3347 // prefetch encodings
3348
3349 enc_class aarch64_enc_prefetchw(memory mem) %{
3350 Register base = as_Register($mem$$base);
3351 int index = $mem$$index;
3352 int scale = $mem$$scale;
3353 int disp = $mem$$disp;
3354 if (index == -1) {
3355 // Fix up any out-of-range offsets.
3356 assert_different_registers(rscratch1, base);
3357 Address addr = Address(base, disp);
3358 addr = __ legitimize_address(addr, 8, rscratch1);
3359 __ prfm(addr, PSTL1KEEP);
3360 } else {
3361 Register index_reg = as_Register(index);
3362 if (disp == 0) {
3363 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3364 } else {
3365 __ lea(rscratch1, Address(base, disp));
3366 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3367 }
3368 }
3369 %}
3370
3371 // mov encodings
3372
3373 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3374 uint32_t con = (uint32_t)$src$$constant;
3375 Register dst_reg = as_Register($dst$$reg);
3376 if (con == 0) {
3377 __ movw(dst_reg, zr);
3378 } else {
3379 __ movw(dst_reg, con);
3380 }
3381 %}
3382
3383 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3384 Register dst_reg = as_Register($dst$$reg);
3385 uint64_t con = (uint64_t)$src$$constant;
3386 if (con == 0) {
3387 __ mov(dst_reg, zr);
3388 } else {
3389 __ mov(dst_reg, con);
3390 }
3391 %}
3392
3393 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3394 Register dst_reg = as_Register($dst$$reg);
3395 address con = (address)$src$$constant;
3396 if (con == nullptr || con == (address)1) {
3397 ShouldNotReachHere();
3398 } else {
3399 relocInfo::relocType rtype = $src->constant_reloc();
3400 if (rtype == relocInfo::oop_type) {
3401 __ movoop(dst_reg, (jobject)con);
3402 } else if (rtype == relocInfo::metadata_type) {
3403 __ mov_metadata(dst_reg, (Metadata*)con);
3404 } else {
3405 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3406 // load fake address constants using a normal move
3407 if (! __ is_valid_AArch64_address(con) ||
3408 con < (address)(uintptr_t)os::vm_page_size() ||
3409 rtype == relocInfo::none) {
3410 __ mov(dst_reg, con);
3411 } else {
3412 // use shorter adrp/add sequence for external_word relocation
3413 uint64_t offset;
3414 __ adrp(dst_reg, Address(con, rtype), offset);
3415 __ add(dst_reg, dst_reg, offset);
3416 }
3417 }
3418 }
3419 %}
3420
3421 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3422 Register dst_reg = as_Register($dst$$reg);
3423 __ mov(dst_reg, zr);
3424 %}
3425
3426 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3427 Register dst_reg = as_Register($dst$$reg);
3428 __ mov(dst_reg, (uint64_t)1);
3429 %}
3430
3431 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3432 Register dst_reg = as_Register($dst$$reg);
3433 address con = (address)$src$$constant;
3434 if (con == nullptr) {
3435 ShouldNotReachHere();
3436 } else {
3437 relocInfo::relocType rtype = $src->constant_reloc();
3438 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3439 __ set_narrow_oop(dst_reg, (jobject)con);
3440 }
3441 %}
3442
3443 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3444 Register dst_reg = as_Register($dst$$reg);
3445 __ mov(dst_reg, zr);
3446 %}
3447
3448 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3449 Register dst_reg = as_Register($dst$$reg);
3450 address con = (address)$src$$constant;
3451 if (con == nullptr) {
3452 ShouldNotReachHere();
3453 } else {
3454 relocInfo::relocType rtype = $src->constant_reloc();
3455 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3456 __ set_narrow_klass(dst_reg, (Klass *)con);
3457 }
3458 %}
3459
3460 // arithmetic encodings
3461
3462 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3463 Register dst_reg = as_Register($dst$$reg);
3464 Register src_reg = as_Register($src1$$reg);
3465 int32_t con = (int32_t)$src2$$constant;
3466 // add has primary == 0, subtract has primary == 1
3467 if ($primary) { con = -con; }
3468 if (con < 0) {
3469 __ subw(dst_reg, src_reg, -con);
3470 } else {
3471 __ addw(dst_reg, src_reg, con);
3472 }
3473 %}
3474
3475 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3476 Register dst_reg = as_Register($dst$$reg);
3477 Register src_reg = as_Register($src1$$reg);
3478 int32_t con = (int32_t)$src2$$constant;
3479 // add has primary == 0, subtract has primary == 1
3480 if ($primary) { con = -con; }
3481 if (con < 0) {
3482 __ sub(dst_reg, src_reg, -con);
3483 } else {
3484 __ add(dst_reg, src_reg, con);
3485 }
3486 %}
3487
3488 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3489 Register dst_reg = as_Register($dst$$reg);
3490 Register src1_reg = as_Register($src1$$reg);
3491 Register src2_reg = as_Register($src2$$reg);
3492 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3493 %}
3494
3495 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3496 Register dst_reg = as_Register($dst$$reg);
3497 Register src1_reg = as_Register($src1$$reg);
3498 Register src2_reg = as_Register($src2$$reg);
3499 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3500 %}
3501
3502 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3503 Register dst_reg = as_Register($dst$$reg);
3504 Register src1_reg = as_Register($src1$$reg);
3505 Register src2_reg = as_Register($src2$$reg);
3506 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3507 %}
3508
3509 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3510 Register dst_reg = as_Register($dst$$reg);
3511 Register src1_reg = as_Register($src1$$reg);
3512 Register src2_reg = as_Register($src2$$reg);
3513 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3514 %}
3515
3516 // compare instruction encodings
3517
3518 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3519 Register reg1 = as_Register($src1$$reg);
3520 Register reg2 = as_Register($src2$$reg);
3521 __ cmpw(reg1, reg2);
3522 %}
3523
3524 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3525 Register reg = as_Register($src1$$reg);
3526 int32_t val = $src2$$constant;
3527 if (val >= 0) {
3528 __ subsw(zr, reg, val);
3529 } else {
3530 __ addsw(zr, reg, -val);
3531 }
3532 %}
3533
3534 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3535 Register reg1 = as_Register($src1$$reg);
3536 uint32_t val = (uint32_t)$src2$$constant;
3537 __ movw(rscratch1, val);
3538 __ cmpw(reg1, rscratch1);
3539 %}
3540
3541 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3542 Register reg1 = as_Register($src1$$reg);
3543 Register reg2 = as_Register($src2$$reg);
3544 __ cmp(reg1, reg2);
3545 %}
3546
3547 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3548 Register reg = as_Register($src1$$reg);
3549 int64_t val = $src2$$constant;
3550 if (val >= 0) {
3551 __ subs(zr, reg, val);
3552 } else if (val != -val) {
3553 __ adds(zr, reg, -val);
3554 } else {
3555 // aargh, Long.MIN_VALUE is a special case
3556 __ orr(rscratch1, zr, (uint64_t)val);
3557 __ subs(zr, reg, rscratch1);
3558 }
3559 %}
3560
3561 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3562 Register reg1 = as_Register($src1$$reg);
3563 uint64_t val = (uint64_t)$src2$$constant;
3564 __ mov(rscratch1, val);
3565 __ cmp(reg1, rscratch1);
3566 %}
3567
3568 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3569 Register reg1 = as_Register($src1$$reg);
3570 Register reg2 = as_Register($src2$$reg);
3571 __ cmp(reg1, reg2);
3572 %}
3573
3574 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3575 Register reg1 = as_Register($src1$$reg);
3576 Register reg2 = as_Register($src2$$reg);
3577 __ cmpw(reg1, reg2);
3578 %}
3579
3580 enc_class aarch64_enc_testp(iRegP src) %{
3581 Register reg = as_Register($src$$reg);
3582 __ cmp(reg, zr);
3583 %}
3584
3585 enc_class aarch64_enc_testn(iRegN src) %{
3586 Register reg = as_Register($src$$reg);
3587 __ cmpw(reg, zr);
3588 %}
3589
3590 enc_class aarch64_enc_b(label lbl) %{
3591 Label *L = $lbl$$label;
3592 __ b(*L);
3593 %}
3594
3595 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3596 Label *L = $lbl$$label;
3597 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3598 %}
3599
3600 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3601 Label *L = $lbl$$label;
3602 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3603 %}
3604
3605 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3606 %{
3607 Register sub_reg = as_Register($sub$$reg);
3608 Register super_reg = as_Register($super$$reg);
3609 Register temp_reg = as_Register($temp$$reg);
3610 Register result_reg = as_Register($result$$reg);
3611
3612 Label miss;
3613 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3614 nullptr, &miss,
3615 /*set_cond_codes:*/ true);
3616 if ($primary) {
3617 __ mov(result_reg, zr);
3618 }
3619 __ bind(miss);
3620 %}
3621
3622 enc_class aarch64_enc_java_static_call(method meth) %{
3623 address addr = (address)$meth$$method;
3624 address call;
3625 if (!_method) {
3626 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3627 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3628 if (call == nullptr) {
3629 ciEnv::current()->record_failure("CodeCache is full");
3630 return;
3631 }
3632 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3633 // The NOP here is purely to ensure that eliding a call to
3634 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3635 __ nop();
3636 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3637 } else {
3638 int method_index = resolved_method_index(masm);
3639 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3640 : static_call_Relocation::spec(method_index);
3641 call = __ trampoline_call(Address(addr, rspec));
3642 if (call == nullptr) {
3643 ciEnv::current()->record_failure("CodeCache is full");
3644 return;
3645 }
3646 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3647 // Calls of the same statically bound method can share
3648 // a stub to the interpreter.
3649 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3650 } else {
3651 // Emit stub for static call
3652 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3653 if (stub == nullptr) {
3654 ciEnv::current()->record_failure("CodeCache is full");
3655 return;
3656 }
3657 }
3658 }
3659
3660 __ post_call_nop();
3661
3662 // Only non uncommon_trap calls need to reinitialize ptrue.
3663 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3664 __ reinitialize_ptrue();
3665 }
3666 %}
3667
3668 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3669 int method_index = resolved_method_index(masm);
3670 address call = __ ic_call((address)$meth$$method, method_index);
3671 if (call == nullptr) {
3672 ciEnv::current()->record_failure("CodeCache is full");
3673 return;
3674 }
3675 __ post_call_nop();
3676 if (Compile::current()->max_vector_size() > 0) {
3677 __ reinitialize_ptrue();
3678 }
3679 %}
3680
3681 enc_class aarch64_enc_call_epilog() %{
3682 if (VerifyStackAtCalls) {
3683 // Check that stack depth is unchanged: find majik cookie on stack
3684 __ call_Unimplemented();
3685 }
3686 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3687 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3688 // Search for the corresponding projection, get the register and emit code that initializes it.
3689 uint con = (tf()->range_cc()->cnt() - 1);
3690 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3691 ProjNode* proj = fast_out(i)->as_Proj();
3692 if (proj->_con == con) {
3693 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3694 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3695 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3696 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3697 __ cmp(r0, zr);
3698 __ cset(toReg, Assembler::NE);
3699 if (reg->is_stack()) {
3700 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3701 __ str(toReg, Address(sp, st_off));
3702 }
3703 break;
3704 }
3705 }
3706 if (return_value_is_used()) {
3707 // An inline type is returned as fields in multiple registers.
3708 // R0 either contains an oop if the inline type is buffered or a pointer
3709 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3710 // if the lowest bit is set to allow C2 to use the oop after null checking.
3711 // r0 &= (r0 & 1) - 1
3712 __ andr(rscratch1, r0, 0x1);
3713 __ sub(rscratch1, rscratch1, 0x1);
3714 __ andr(r0, r0, rscratch1);
3715 }
3716 }
3717 %}
3718
3719 enc_class aarch64_enc_java_to_runtime(method meth) %{
3720 // some calls to generated routines (arraycopy code) are scheduled
3721 // by C2 as runtime calls. if so we can call them using a br (they
3722 // will be in a reachable segment) otherwise we have to use a blr
3723 // which loads the absolute address into a register.
3724 address entry = (address)$meth$$method;
3725 CodeBlob *cb = CodeCache::find_blob(entry);
3726 if (cb) {
3727 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3728 if (call == nullptr) {
3729 ciEnv::current()->record_failure("CodeCache is full");
3730 return;
3731 }
3732 __ post_call_nop();
3733 } else {
3734 Label retaddr;
3735 // Make the anchor frame walkable
3736 __ adr(rscratch2, retaddr);
3737 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3738 __ lea(rscratch1, RuntimeAddress(entry));
3739 __ blr(rscratch1);
3740 __ bind(retaddr);
3741 __ post_call_nop();
3742 }
3743 if (Compile::current()->max_vector_size() > 0) {
3744 __ reinitialize_ptrue();
3745 }
3746 %}
3747
3748 enc_class aarch64_enc_rethrow() %{
3749 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3750 %}
3751
3752 enc_class aarch64_enc_ret() %{
3753 #ifdef ASSERT
3754 if (Compile::current()->max_vector_size() > 0) {
3755 __ verify_ptrue();
3756 }
3757 #endif
3758 __ ret(lr);
3759 %}
3760
3761 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3762 Register target_reg = as_Register($jump_target$$reg);
3763 __ br(target_reg);
3764 %}
3765
3766 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3767 Register target_reg = as_Register($jump_target$$reg);
3768 // exception oop should be in r0
3769 // ret addr has been popped into lr
3770 // callee expects it in r3
3771 __ mov(r3, lr);
3772 __ br(target_reg);
3773 %}
3774
3775 %}
3776
3777 //----------FRAME--------------------------------------------------------------
3778 // Definition of frame structure and management information.
3779 //
3780 // S T A C K L A Y O U T Allocators stack-slot number
3781 // | (to get allocators register number
3782 // G Owned by | | v add OptoReg::stack0())
3783 // r CALLER | |
3784 // o | +--------+ pad to even-align allocators stack-slot
3785 // w V | pad0 | numbers; owned by CALLER
3786 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3787 // h ^ | in | 5
3788 // | | args | 4 Holes in incoming args owned by SELF
3789 // | | | | 3
3790 // | | +--------+
3791 // V | | old out| Empty on Intel, window on Sparc
3792 // | old |preserve| Must be even aligned.
3793 // | SP-+--------+----> Matcher::_old_SP, even aligned
3794 // | | in | 3 area for Intel ret address
3795 // Owned by |preserve| Empty on Sparc.
3796 // SELF +--------+
3797 // | | pad2 | 2 pad to align old SP
3798 // | +--------+ 1
3799 // | | locks | 0
3800 // | +--------+----> OptoReg::stack0(), even aligned
3801 // | | pad1 | 11 pad to align new SP
3802 // | +--------+
3803 // | | | 10
3804 // | | spills | 9 spills
3805 // V | | 8 (pad0 slot for callee)
3806 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3807 // ^ | out | 7
3808 // | | args | 6 Holes in outgoing args owned by CALLEE
3809 // Owned by +--------+
3810 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3811 // | new |preserve| Must be even-aligned.
3812 // | SP-+--------+----> Matcher::_new_SP, even aligned
3813 // | | |
3814 //
3815 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3816 // known from SELF's arguments and the Java calling convention.
3817 // Region 6-7 is determined per call site.
3818 // Note 2: If the calling convention leaves holes in the incoming argument
3819 // area, those holes are owned by SELF. Holes in the outgoing area
3820 // are owned by the CALLEE. Holes should not be necessary in the
3821 // incoming area, as the Java calling convention is completely under
3822 // the control of the AD file. Doubles can be sorted and packed to
3823 // avoid holes. Holes in the outgoing arguments may be necessary for
3824 // varargs C calling conventions.
3825 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3826 // even aligned with pad0 as needed.
3827 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3828 // (the latter is true on Intel but is it false on AArch64?)
3829 // region 6-11 is even aligned; it may be padded out more so that
3830 // the region from SP to FP meets the minimum stack alignment.
3831 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3832 // alignment. Region 11, pad1, may be dynamically extended so that
3833 // SP meets the minimum alignment.
3834
3835 frame %{
3836 // These three registers define part of the calling convention
3837 // between compiled code and the interpreter.
3838
3839 // Inline Cache Register or Method for I2C.
3840 inline_cache_reg(R12);
3841
3842 // Number of stack slots consumed by locking an object
3843 sync_stack_slots(2);
3844
3845 // Compiled code's Frame Pointer
3846 frame_pointer(R31);
3847
3848 // Stack alignment requirement
3849 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3850
3851 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3852 // for calls to C. Supports the var-args backing area for register parms.
3853 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3854
3855 // The after-PROLOG location of the return address. Location of
3856 // return address specifies a type (REG or STACK) and a number
3857 // representing the register number (i.e. - use a register name) or
3858 // stack slot.
3859 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3860 // Otherwise, it is above the locks and verification slot and alignment word
3861 // TODO this may well be correct but need to check why that - 2 is there
3862 // ppc port uses 0 but we definitely need to allow for fixed_slots
3863 // which folds in the space used for monitors
3864 return_addr(STACK - 2 +
3865 align_up((Compile::current()->in_preserve_stack_slots() +
3866 Compile::current()->fixed_slots()),
3867 stack_alignment_in_slots()));
3868
3869 // Location of compiled Java return values. Same as C for now.
3870 return_value
3871 %{
3872 // TODO do we allow ideal_reg == Op_RegN???
3873 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3874 "only return normal values");
3875
3876 static const int lo[Op_RegL + 1] = { // enum name
3877 0, // Op_Node
3878 0, // Op_Set
3879 R0_num, // Op_RegN
3880 R0_num, // Op_RegI
3881 R0_num, // Op_RegP
3882 V0_num, // Op_RegF
3883 V0_num, // Op_RegD
3884 R0_num // Op_RegL
3885 };
3886
3887 static const int hi[Op_RegL + 1] = { // enum name
3888 0, // Op_Node
3889 0, // Op_Set
3890 OptoReg::Bad, // Op_RegN
3891 OptoReg::Bad, // Op_RegI
3892 R0_H_num, // Op_RegP
3893 OptoReg::Bad, // Op_RegF
3894 V0_H_num, // Op_RegD
3895 R0_H_num // Op_RegL
3896 };
3897
3898 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3899 %}
3900 %}
3901
3902 //----------ATTRIBUTES---------------------------------------------------------
3903 //----------Operand Attributes-------------------------------------------------
3904 op_attrib op_cost(1); // Required cost attribute
3905
3906 //----------Instruction Attributes---------------------------------------------
3907 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3908 ins_attrib ins_size(32); // Required size attribute (in bits)
3909 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3910 // a non-matching short branch variant
3911 // of some long branch?
3912 ins_attrib ins_alignment(4); // Required alignment attribute (must
3913 // be a power of 2) specifies the
3914 // alignment that some part of the
3915 // instruction (not necessarily the
3916 // start) requires. If > 1, a
3917 // compute_padding() function must be
3918 // provided for the instruction
3919
3920 // Whether this node is expanded during code emission into a sequence of
3921 // instructions and the first instruction can perform an implicit null check.
3922 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3923
3924 //----------OPERANDS-----------------------------------------------------------
3925 // Operand definitions must precede instruction definitions for correct parsing
3926 // in the ADLC because operands constitute user defined types which are used in
3927 // instruction definitions.
3928
3929 //----------Simple Operands----------------------------------------------------
3930
3931 // Integer operands 32 bit
3932 // 32 bit immediate
3933 operand immI()
3934 %{
3935 match(ConI);
3936
3937 op_cost(0);
3938 format %{ %}
3939 interface(CONST_INTER);
3940 %}
3941
3942 // 32 bit zero
3943 operand immI0()
3944 %{
3945 predicate(n->get_int() == 0);
3946 match(ConI);
3947
3948 op_cost(0);
3949 format %{ %}
3950 interface(CONST_INTER);
3951 %}
3952
3953 // 32 bit unit increment
3954 operand immI_1()
3955 %{
3956 predicate(n->get_int() == 1);
3957 match(ConI);
3958
3959 op_cost(0);
3960 format %{ %}
3961 interface(CONST_INTER);
3962 %}
3963
3964 // 32 bit unit decrement
3965 operand immI_M1()
3966 %{
3967 predicate(n->get_int() == -1);
3968 match(ConI);
3969
3970 op_cost(0);
3971 format %{ %}
3972 interface(CONST_INTER);
3973 %}
3974
3975 // Shift values for add/sub extension shift
3976 operand immIExt()
3977 %{
3978 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3979 match(ConI);
3980
3981 op_cost(0);
3982 format %{ %}
3983 interface(CONST_INTER);
3984 %}
3985
3986 operand immI_gt_1()
3987 %{
3988 predicate(n->get_int() > 1);
3989 match(ConI);
3990
3991 op_cost(0);
3992 format %{ %}
3993 interface(CONST_INTER);
3994 %}
3995
3996 operand immI_le_4()
3997 %{
3998 predicate(n->get_int() <= 4);
3999 match(ConI);
4000
4001 op_cost(0);
4002 format %{ %}
4003 interface(CONST_INTER);
4004 %}
4005
4006 operand immI_4()
4007 %{
4008 predicate(n->get_int() == 4);
4009 match(ConI);
4010
4011 op_cost(0);
4012 format %{ %}
4013 interface(CONST_INTER);
4014 %}
4015
4016 operand immI_16()
4017 %{
4018 predicate(n->get_int() == 16);
4019 match(ConI);
4020
4021 op_cost(0);
4022 format %{ %}
4023 interface(CONST_INTER);
4024 %}
4025
4026 operand immI_24()
4027 %{
4028 predicate(n->get_int() == 24);
4029 match(ConI);
4030
4031 op_cost(0);
4032 format %{ %}
4033 interface(CONST_INTER);
4034 %}
4035
4036 operand immI_32()
4037 %{
4038 predicate(n->get_int() == 32);
4039 match(ConI);
4040
4041 op_cost(0);
4042 format %{ %}
4043 interface(CONST_INTER);
4044 %}
4045
4046 operand immI_48()
4047 %{
4048 predicate(n->get_int() == 48);
4049 match(ConI);
4050
4051 op_cost(0);
4052 format %{ %}
4053 interface(CONST_INTER);
4054 %}
4055
4056 operand immI_56()
4057 %{
4058 predicate(n->get_int() == 56);
4059 match(ConI);
4060
4061 op_cost(0);
4062 format %{ %}
4063 interface(CONST_INTER);
4064 %}
4065
4066 operand immI_255()
4067 %{
4068 predicate(n->get_int() == 255);
4069 match(ConI);
4070
4071 op_cost(0);
4072 format %{ %}
4073 interface(CONST_INTER);
4074 %}
4075
4076 operand immI_65535()
4077 %{
4078 predicate(n->get_int() == 65535);
4079 match(ConI);
4080
4081 op_cost(0);
4082 format %{ %}
4083 interface(CONST_INTER);
4084 %}
4085
4086 operand immI_positive()
4087 %{
4088 predicate(n->get_int() > 0);
4089 match(ConI);
4090
4091 op_cost(0);
4092 format %{ %}
4093 interface(CONST_INTER);
4094 %}
4095
4096 // BoolTest condition for signed compare
4097 operand immI_cmp_cond()
4098 %{
4099 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4100 match(ConI);
4101
4102 op_cost(0);
4103 format %{ %}
4104 interface(CONST_INTER);
4105 %}
4106
4107 // BoolTest condition for unsigned compare
4108 operand immI_cmpU_cond()
4109 %{
4110 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4111 match(ConI);
4112
4113 op_cost(0);
4114 format %{ %}
4115 interface(CONST_INTER);
4116 %}
4117
4118 operand immL_255()
4119 %{
4120 predicate(n->get_long() == 255L);
4121 match(ConL);
4122
4123 op_cost(0);
4124 format %{ %}
4125 interface(CONST_INTER);
4126 %}
4127
4128 operand immL_65535()
4129 %{
4130 predicate(n->get_long() == 65535L);
4131 match(ConL);
4132
4133 op_cost(0);
4134 format %{ %}
4135 interface(CONST_INTER);
4136 %}
4137
4138 operand immL_4294967295()
4139 %{
4140 predicate(n->get_long() == 4294967295L);
4141 match(ConL);
4142
4143 op_cost(0);
4144 format %{ %}
4145 interface(CONST_INTER);
4146 %}
4147
4148 operand immL_bitmask()
4149 %{
4150 predicate((n->get_long() != 0)
4151 && ((n->get_long() & 0xc000000000000000l) == 0)
4152 && is_power_of_2(n->get_long() + 1));
4153 match(ConL);
4154
4155 op_cost(0);
4156 format %{ %}
4157 interface(CONST_INTER);
4158 %}
4159
4160 operand immI_bitmask()
4161 %{
4162 predicate((n->get_int() != 0)
4163 && ((n->get_int() & 0xc0000000) == 0)
4164 && is_power_of_2(n->get_int() + 1));
4165 match(ConI);
4166
4167 op_cost(0);
4168 format %{ %}
4169 interface(CONST_INTER);
4170 %}
4171
4172 operand immL_positive_bitmaskI()
4173 %{
4174 predicate((n->get_long() != 0)
4175 && ((julong)n->get_long() < 0x80000000ULL)
4176 && is_power_of_2(n->get_long() + 1));
4177 match(ConL);
4178
4179 op_cost(0);
4180 format %{ %}
4181 interface(CONST_INTER);
4182 %}
4183
4184 // Scale values for scaled offset addressing modes (up to long but not quad)
4185 operand immIScale()
4186 %{
4187 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4188 match(ConI);
4189
4190 op_cost(0);
4191 format %{ %}
4192 interface(CONST_INTER);
4193 %}
4194
4195 // 5 bit signed integer
4196 operand immI5()
4197 %{
4198 predicate(Assembler::is_simm(n->get_int(), 5));
4199 match(ConI);
4200
4201 op_cost(0);
4202 format %{ %}
4203 interface(CONST_INTER);
4204 %}
4205
4206 // 7 bit unsigned integer
4207 operand immIU7()
4208 %{
4209 predicate(Assembler::is_uimm(n->get_int(), 7));
4210 match(ConI);
4211
4212 op_cost(0);
4213 format %{ %}
4214 interface(CONST_INTER);
4215 %}
4216
4217 // Offset for scaled or unscaled immediate loads and stores
4218 operand immIOffset()
4219 %{
4220 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4221 match(ConI);
4222
4223 op_cost(0);
4224 format %{ %}
4225 interface(CONST_INTER);
4226 %}
4227
4228 operand immIOffset1()
4229 %{
4230 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4231 match(ConI);
4232
4233 op_cost(0);
4234 format %{ %}
4235 interface(CONST_INTER);
4236 %}
4237
4238 operand immIOffset2()
4239 %{
4240 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4241 match(ConI);
4242
4243 op_cost(0);
4244 format %{ %}
4245 interface(CONST_INTER);
4246 %}
4247
4248 operand immIOffset4()
4249 %{
4250 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4251 match(ConI);
4252
4253 op_cost(0);
4254 format %{ %}
4255 interface(CONST_INTER);
4256 %}
4257
4258 operand immIOffset8()
4259 %{
4260 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4261 match(ConI);
4262
4263 op_cost(0);
4264 format %{ %}
4265 interface(CONST_INTER);
4266 %}
4267
4268 operand immIOffset16()
4269 %{
4270 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4271 match(ConI);
4272
4273 op_cost(0);
4274 format %{ %}
4275 interface(CONST_INTER);
4276 %}
4277
4278 operand immLOffset()
4279 %{
4280 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4281 match(ConL);
4282
4283 op_cost(0);
4284 format %{ %}
4285 interface(CONST_INTER);
4286 %}
4287
4288 operand immLoffset1()
4289 %{
4290 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4291 match(ConL);
4292
4293 op_cost(0);
4294 format %{ %}
4295 interface(CONST_INTER);
4296 %}
4297
4298 operand immLoffset2()
4299 %{
4300 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4301 match(ConL);
4302
4303 op_cost(0);
4304 format %{ %}
4305 interface(CONST_INTER);
4306 %}
4307
4308 operand immLoffset4()
4309 %{
4310 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4311 match(ConL);
4312
4313 op_cost(0);
4314 format %{ %}
4315 interface(CONST_INTER);
4316 %}
4317
4318 operand immLoffset8()
4319 %{
4320 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4321 match(ConL);
4322
4323 op_cost(0);
4324 format %{ %}
4325 interface(CONST_INTER);
4326 %}
4327
4328 operand immLoffset16()
4329 %{
4330 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4331 match(ConL);
4332
4333 op_cost(0);
4334 format %{ %}
4335 interface(CONST_INTER);
4336 %}
4337
4338 // 5 bit signed long integer
4339 operand immL5()
4340 %{
4341 predicate(Assembler::is_simm(n->get_long(), 5));
4342 match(ConL);
4343
4344 op_cost(0);
4345 format %{ %}
4346 interface(CONST_INTER);
4347 %}
4348
4349 // 7 bit unsigned long integer
4350 operand immLU7()
4351 %{
4352 predicate(Assembler::is_uimm(n->get_long(), 7));
4353 match(ConL);
4354
4355 op_cost(0);
4356 format %{ %}
4357 interface(CONST_INTER);
4358 %}
4359
4360 // 8 bit signed value.
4361 operand immI8()
4362 %{
4363 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4364 match(ConI);
4365
4366 op_cost(0);
4367 format %{ %}
4368 interface(CONST_INTER);
4369 %}
4370
4371 // 8 bit signed value (simm8), or #simm8 LSL 8.
4372 operand immIDupV()
4373 %{
4374 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4375 match(ConI);
4376
4377 op_cost(0);
4378 format %{ %}
4379 interface(CONST_INTER);
4380 %}
4381
4382 // 8 bit signed value (simm8), or #simm8 LSL 8.
4383 operand immLDupV()
4384 %{
4385 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4386 match(ConL);
4387
4388 op_cost(0);
4389 format %{ %}
4390 interface(CONST_INTER);
4391 %}
4392
4393 // 8 bit signed value (simm8), or #simm8 LSL 8.
4394 operand immHDupV()
4395 %{
4396 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4397 match(ConH);
4398
4399 op_cost(0);
4400 format %{ %}
4401 interface(CONST_INTER);
4402 %}
4403
4404 // 8 bit integer valid for vector add sub immediate
4405 operand immBAddSubV()
4406 %{
4407 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4408 match(ConI);
4409
4410 op_cost(0);
4411 format %{ %}
4412 interface(CONST_INTER);
4413 %}
4414
4415 // 32 bit integer valid for add sub immediate
4416 operand immIAddSub()
4417 %{
4418 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4419 match(ConI);
4420 op_cost(0);
4421 format %{ %}
4422 interface(CONST_INTER);
4423 %}
4424
4425 // 32 bit integer valid for vector add sub immediate
4426 operand immIAddSubV()
4427 %{
4428 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4429 match(ConI);
4430
4431 op_cost(0);
4432 format %{ %}
4433 interface(CONST_INTER);
4434 %}
4435
4436 // 32 bit unsigned integer valid for logical immediate
4437
4438 operand immBLog()
4439 %{
4440 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4441 match(ConI);
4442
4443 op_cost(0);
4444 format %{ %}
4445 interface(CONST_INTER);
4446 %}
4447
4448 operand immSLog()
4449 %{
4450 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4451 match(ConI);
4452
4453 op_cost(0);
4454 format %{ %}
4455 interface(CONST_INTER);
4456 %}
4457
4458 operand immILog()
4459 %{
4460 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4461 match(ConI);
4462
4463 op_cost(0);
4464 format %{ %}
4465 interface(CONST_INTER);
4466 %}
4467
4468 // Integer operands 64 bit
4469 // 64 bit immediate
4470 operand immL()
4471 %{
4472 match(ConL);
4473
4474 op_cost(0);
4475 format %{ %}
4476 interface(CONST_INTER);
4477 %}
4478
4479 // 64 bit zero
4480 operand immL0()
4481 %{
4482 predicate(n->get_long() == 0);
4483 match(ConL);
4484
4485 op_cost(0);
4486 format %{ %}
4487 interface(CONST_INTER);
4488 %}
4489
4490 // 64 bit unit decrement
4491 operand immL_M1()
4492 %{
4493 predicate(n->get_long() == -1);
4494 match(ConL);
4495
4496 op_cost(0);
4497 format %{ %}
4498 interface(CONST_INTER);
4499 %}
4500
4501 // 64 bit integer valid for add sub immediate
4502 operand immLAddSub()
4503 %{
4504 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4505 match(ConL);
4506 op_cost(0);
4507 format %{ %}
4508 interface(CONST_INTER);
4509 %}
4510
4511 // 64 bit integer valid for addv subv immediate
4512 operand immLAddSubV()
4513 %{
4514 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4515 match(ConL);
4516
4517 op_cost(0);
4518 format %{ %}
4519 interface(CONST_INTER);
4520 %}
4521
4522 // 64 bit integer valid for logical immediate
4523 operand immLLog()
4524 %{
4525 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4526 match(ConL);
4527 op_cost(0);
4528 format %{ %}
4529 interface(CONST_INTER);
4530 %}
4531
4532 // Long Immediate: low 32-bit mask
4533 operand immL_32bits()
4534 %{
4535 predicate(n->get_long() == 0xFFFFFFFFL);
4536 match(ConL);
4537 op_cost(0);
4538 format %{ %}
4539 interface(CONST_INTER);
4540 %}
4541
4542 // Pointer operands
4543 // Pointer Immediate
4544 operand immP()
4545 %{
4546 match(ConP);
4547
4548 op_cost(0);
4549 format %{ %}
4550 interface(CONST_INTER);
4551 %}
4552
4553 // nullptr Pointer Immediate
4554 operand immP0()
4555 %{
4556 predicate(n->get_ptr() == 0);
4557 match(ConP);
4558
4559 op_cost(0);
4560 format %{ %}
4561 interface(CONST_INTER);
4562 %}
4563
4564 // Pointer Immediate One
4565 // this is used in object initialization (initial object header)
4566 operand immP_1()
4567 %{
4568 predicate(n->get_ptr() == 1);
4569 match(ConP);
4570
4571 op_cost(0);
4572 format %{ %}
4573 interface(CONST_INTER);
4574 %}
4575
4576 // AOT Runtime Constants Address
4577 operand immAOTRuntimeConstantsAddress()
4578 %{
4579 // Check if the address is in the range of AOT Runtime Constants
4580 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4581 match(ConP);
4582
4583 op_cost(0);
4584 format %{ %}
4585 interface(CONST_INTER);
4586 %}
4587
4588 // Float and Double operands
4589 // Double Immediate
4590 operand immD()
4591 %{
4592 match(ConD);
4593 op_cost(0);
4594 format %{ %}
4595 interface(CONST_INTER);
4596 %}
4597
4598 // Double Immediate: +0.0d
4599 operand immD0()
4600 %{
4601 predicate(jlong_cast(n->getd()) == 0);
4602 match(ConD);
4603
4604 op_cost(0);
4605 format %{ %}
4606 interface(CONST_INTER);
4607 %}
4608
4609 // constant 'double +0.0'.
4610 operand immDPacked()
4611 %{
4612 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4613 match(ConD);
4614 op_cost(0);
4615 format %{ %}
4616 interface(CONST_INTER);
4617 %}
4618
4619 // Float Immediate
4620 operand immF()
4621 %{
4622 match(ConF);
4623 op_cost(0);
4624 format %{ %}
4625 interface(CONST_INTER);
4626 %}
4627
4628 // Float Immediate: +0.0f.
4629 operand immF0()
4630 %{
4631 predicate(jint_cast(n->getf()) == 0);
4632 match(ConF);
4633
4634 op_cost(0);
4635 format %{ %}
4636 interface(CONST_INTER);
4637 %}
4638
4639 // Half Float (FP16) Immediate
4640 operand immH()
4641 %{
4642 match(ConH);
4643 op_cost(0);
4644 format %{ %}
4645 interface(CONST_INTER);
4646 %}
4647
4648 //
4649 operand immFPacked()
4650 %{
4651 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4652 match(ConF);
4653 op_cost(0);
4654 format %{ %}
4655 interface(CONST_INTER);
4656 %}
4657
4658 // Narrow pointer operands
4659 // Narrow Pointer Immediate
4660 operand immN()
4661 %{
4662 match(ConN);
4663
4664 op_cost(0);
4665 format %{ %}
4666 interface(CONST_INTER);
4667 %}
4668
4669 // Narrow nullptr Pointer Immediate
4670 operand immN0()
4671 %{
4672 predicate(n->get_narrowcon() == 0);
4673 match(ConN);
4674
4675 op_cost(0);
4676 format %{ %}
4677 interface(CONST_INTER);
4678 %}
4679
4680 operand immNKlass()
4681 %{
4682 match(ConNKlass);
4683
4684 op_cost(0);
4685 format %{ %}
4686 interface(CONST_INTER);
4687 %}
4688
4689 // Integer 32 bit Register Operands
4690 // Integer 32 bitRegister (excludes SP)
4691 operand iRegI()
4692 %{
4693 constraint(ALLOC_IN_RC(any_reg32));
4694 match(RegI);
4695 match(iRegINoSp);
4696 op_cost(0);
4697 format %{ %}
4698 interface(REG_INTER);
4699 %}
4700
4701 // Integer 32 bit Register not Special
4702 operand iRegINoSp()
4703 %{
4704 constraint(ALLOC_IN_RC(no_special_reg32));
4705 match(RegI);
4706 op_cost(0);
4707 format %{ %}
4708 interface(REG_INTER);
4709 %}
4710
4711 // Integer 64 bit Register Operands
4712 // Integer 64 bit Register (includes SP)
4713 operand iRegL()
4714 %{
4715 constraint(ALLOC_IN_RC(any_reg));
4716 match(RegL);
4717 match(iRegLNoSp);
4718 op_cost(0);
4719 format %{ %}
4720 interface(REG_INTER);
4721 %}
4722
4723 // Integer 64 bit Register not Special
4724 operand iRegLNoSp()
4725 %{
4726 constraint(ALLOC_IN_RC(no_special_reg));
4727 match(RegL);
4728 match(iRegL_R0);
4729 format %{ %}
4730 interface(REG_INTER);
4731 %}
4732
4733 // Pointer Register Operands
4734 // Pointer Register
4735 operand iRegP()
4736 %{
4737 constraint(ALLOC_IN_RC(ptr_reg));
4738 match(RegP);
4739 match(iRegPNoSp);
4740 match(iRegP_R0);
4741 //match(iRegP_R2);
4742 //match(iRegP_R4);
4743 match(iRegP_R5);
4744 match(thread_RegP);
4745 op_cost(0);
4746 format %{ %}
4747 interface(REG_INTER);
4748 %}
4749
4750 // Pointer 64 bit Register not Special
4751 operand iRegPNoSp()
4752 %{
4753 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4754 match(RegP);
4755 // match(iRegP);
4756 // match(iRegP_R0);
4757 // match(iRegP_R2);
4758 // match(iRegP_R4);
4759 // match(iRegP_R5);
4760 // match(thread_RegP);
4761 op_cost(0);
4762 format %{ %}
4763 interface(REG_INTER);
4764 %}
4765
4766 // This operand is not allowed to use rfp even if
4767 // rfp is not used to hold the frame pointer.
4768 operand iRegPNoSpNoRfp()
4769 %{
4770 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4771 match(RegP);
4772 match(iRegPNoSp);
4773 op_cost(0);
4774 format %{ %}
4775 interface(REG_INTER);
4776 %}
4777
4778 // Pointer 64 bit Register R0 only
4779 operand iRegP_R0()
4780 %{
4781 constraint(ALLOC_IN_RC(r0_reg));
4782 match(RegP);
4783 // match(iRegP);
4784 match(iRegPNoSp);
4785 op_cost(0);
4786 format %{ %}
4787 interface(REG_INTER);
4788 %}
4789
4790 // Pointer 64 bit Register R1 only
4791 operand iRegP_R1()
4792 %{
4793 constraint(ALLOC_IN_RC(r1_reg));
4794 match(RegP);
4795 // match(iRegP);
4796 match(iRegPNoSp);
4797 op_cost(0);
4798 format %{ %}
4799 interface(REG_INTER);
4800 %}
4801
4802 // Pointer 64 bit Register R2 only
4803 operand iRegP_R2()
4804 %{
4805 constraint(ALLOC_IN_RC(r2_reg));
4806 match(RegP);
4807 // match(iRegP);
4808 match(iRegPNoSp);
4809 op_cost(0);
4810 format %{ %}
4811 interface(REG_INTER);
4812 %}
4813
4814 // Pointer 64 bit Register R3 only
4815 operand iRegP_R3()
4816 %{
4817 constraint(ALLOC_IN_RC(r3_reg));
4818 match(RegP);
4819 // match(iRegP);
4820 match(iRegPNoSp);
4821 op_cost(0);
4822 format %{ %}
4823 interface(REG_INTER);
4824 %}
4825
4826 // Pointer 64 bit Register R4 only
4827 operand iRegP_R4()
4828 %{
4829 constraint(ALLOC_IN_RC(r4_reg));
4830 match(RegP);
4831 // match(iRegP);
4832 match(iRegPNoSp);
4833 op_cost(0);
4834 format %{ %}
4835 interface(REG_INTER);
4836 %}
4837
4838 // Pointer 64 bit Register R5 only
4839 operand iRegP_R5()
4840 %{
4841 constraint(ALLOC_IN_RC(r5_reg));
4842 match(RegP);
4843 // match(iRegP);
4844 match(iRegPNoSp);
4845 op_cost(0);
4846 format %{ %}
4847 interface(REG_INTER);
4848 %}
4849
4850 // Pointer 64 bit Register R10 only
4851 operand iRegP_R10()
4852 %{
4853 constraint(ALLOC_IN_RC(r10_reg));
4854 match(RegP);
4855 // match(iRegP);
4856 match(iRegPNoSp);
4857 op_cost(0);
4858 format %{ %}
4859 interface(REG_INTER);
4860 %}
4861
4862 // Long 64 bit Register R0 only
4863 operand iRegL_R0()
4864 %{
4865 constraint(ALLOC_IN_RC(r0_reg));
4866 match(RegL);
4867 match(iRegLNoSp);
4868 op_cost(0);
4869 format %{ %}
4870 interface(REG_INTER);
4871 %}
4872
4873 // Long 64 bit Register R11 only
4874 operand iRegL_R11()
4875 %{
4876 constraint(ALLOC_IN_RC(r11_reg));
4877 match(RegL);
4878 match(iRegLNoSp);
4879 op_cost(0);
4880 format %{ %}
4881 interface(REG_INTER);
4882 %}
4883
4884 // Register R0 only
4885 operand iRegI_R0()
4886 %{
4887 constraint(ALLOC_IN_RC(int_r0_reg));
4888 match(RegI);
4889 match(iRegINoSp);
4890 op_cost(0);
4891 format %{ %}
4892 interface(REG_INTER);
4893 %}
4894
4895 // Register R2 only
4896 operand iRegI_R2()
4897 %{
4898 constraint(ALLOC_IN_RC(int_r2_reg));
4899 match(RegI);
4900 match(iRegINoSp);
4901 op_cost(0);
4902 format %{ %}
4903 interface(REG_INTER);
4904 %}
4905
4906 // Register R3 only
4907 operand iRegI_R3()
4908 %{
4909 constraint(ALLOC_IN_RC(int_r3_reg));
4910 match(RegI);
4911 match(iRegINoSp);
4912 op_cost(0);
4913 format %{ %}
4914 interface(REG_INTER);
4915 %}
4916
4917
4918 // Register R4 only
4919 operand iRegI_R4()
4920 %{
4921 constraint(ALLOC_IN_RC(int_r4_reg));
4922 match(RegI);
4923 match(iRegINoSp);
4924 op_cost(0);
4925 format %{ %}
4926 interface(REG_INTER);
4927 %}
4928
4929
4930 // Pointer Register Operands
4931 // Narrow Pointer Register
4932 operand iRegN()
4933 %{
4934 constraint(ALLOC_IN_RC(any_reg32));
4935 match(RegN);
4936 match(iRegNNoSp);
4937 op_cost(0);
4938 format %{ %}
4939 interface(REG_INTER);
4940 %}
4941
4942 // Integer 64 bit Register not Special
4943 operand iRegNNoSp()
4944 %{
4945 constraint(ALLOC_IN_RC(no_special_reg32));
4946 match(RegN);
4947 op_cost(0);
4948 format %{ %}
4949 interface(REG_INTER);
4950 %}
4951
4952 // Float Register
4953 // Float register operands
4954 operand vRegF()
4955 %{
4956 constraint(ALLOC_IN_RC(float_reg));
4957 match(RegF);
4958
4959 op_cost(0);
4960 format %{ %}
4961 interface(REG_INTER);
4962 %}
4963
4964 // Double Register
4965 // Double register operands
4966 operand vRegD()
4967 %{
4968 constraint(ALLOC_IN_RC(double_reg));
4969 match(RegD);
4970
4971 op_cost(0);
4972 format %{ %}
4973 interface(REG_INTER);
4974 %}
4975
4976 // Generic vector class. This will be used for
4977 // all vector operands, including NEON and SVE.
4978 operand vReg()
4979 %{
4980 constraint(ALLOC_IN_RC(dynamic));
4981 match(VecA);
4982 match(VecD);
4983 match(VecX);
4984
4985 op_cost(0);
4986 format %{ %}
4987 interface(REG_INTER);
4988 %}
4989
4990 operand vReg_V10()
4991 %{
4992 constraint(ALLOC_IN_RC(v10_veca_reg));
4993 match(vReg);
4994
4995 op_cost(0);
4996 format %{ %}
4997 interface(REG_INTER);
4998 %}
4999
5000 operand vReg_V11()
5001 %{
5002 constraint(ALLOC_IN_RC(v11_veca_reg));
5003 match(vReg);
5004
5005 op_cost(0);
5006 format %{ %}
5007 interface(REG_INTER);
5008 %}
5009
5010 operand vReg_V12()
5011 %{
5012 constraint(ALLOC_IN_RC(v12_veca_reg));
5013 match(vReg);
5014
5015 op_cost(0);
5016 format %{ %}
5017 interface(REG_INTER);
5018 %}
5019
5020 operand vReg_V13()
5021 %{
5022 constraint(ALLOC_IN_RC(v13_veca_reg));
5023 match(vReg);
5024
5025 op_cost(0);
5026 format %{ %}
5027 interface(REG_INTER);
5028 %}
5029
5030 operand vReg_V17()
5031 %{
5032 constraint(ALLOC_IN_RC(v17_veca_reg));
5033 match(vReg);
5034
5035 op_cost(0);
5036 format %{ %}
5037 interface(REG_INTER);
5038 %}
5039
5040 operand vReg_V18()
5041 %{
5042 constraint(ALLOC_IN_RC(v18_veca_reg));
5043 match(vReg);
5044
5045 op_cost(0);
5046 format %{ %}
5047 interface(REG_INTER);
5048 %}
5049
5050 operand vReg_V23()
5051 %{
5052 constraint(ALLOC_IN_RC(v23_veca_reg));
5053 match(vReg);
5054
5055 op_cost(0);
5056 format %{ %}
5057 interface(REG_INTER);
5058 %}
5059
5060 operand vReg_V24()
5061 %{
5062 constraint(ALLOC_IN_RC(v24_veca_reg));
5063 match(vReg);
5064
5065 op_cost(0);
5066 format %{ %}
5067 interface(REG_INTER);
5068 %}
5069
5070 operand vecA()
5071 %{
5072 constraint(ALLOC_IN_RC(vectora_reg));
5073 match(VecA);
5074
5075 op_cost(0);
5076 format %{ %}
5077 interface(REG_INTER);
5078 %}
5079
5080 operand vecD()
5081 %{
5082 constraint(ALLOC_IN_RC(vectord_reg));
5083 match(VecD);
5084
5085 op_cost(0);
5086 format %{ %}
5087 interface(REG_INTER);
5088 %}
5089
5090 operand vecX()
5091 %{
5092 constraint(ALLOC_IN_RC(vectorx_reg));
5093 match(VecX);
5094
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand vRegD_V0()
5101 %{
5102 constraint(ALLOC_IN_RC(v0_reg));
5103 match(RegD);
5104 op_cost(0);
5105 format %{ %}
5106 interface(REG_INTER);
5107 %}
5108
5109 operand vRegD_V1()
5110 %{
5111 constraint(ALLOC_IN_RC(v1_reg));
5112 match(RegD);
5113 op_cost(0);
5114 format %{ %}
5115 interface(REG_INTER);
5116 %}
5117
5118 operand vRegD_V2()
5119 %{
5120 constraint(ALLOC_IN_RC(v2_reg));
5121 match(RegD);
5122 op_cost(0);
5123 format %{ %}
5124 interface(REG_INTER);
5125 %}
5126
5127 operand vRegD_V3()
5128 %{
5129 constraint(ALLOC_IN_RC(v3_reg));
5130 match(RegD);
5131 op_cost(0);
5132 format %{ %}
5133 interface(REG_INTER);
5134 %}
5135
5136 operand vRegD_V4()
5137 %{
5138 constraint(ALLOC_IN_RC(v4_reg));
5139 match(RegD);
5140 op_cost(0);
5141 format %{ %}
5142 interface(REG_INTER);
5143 %}
5144
5145 operand vRegD_V5()
5146 %{
5147 constraint(ALLOC_IN_RC(v5_reg));
5148 match(RegD);
5149 op_cost(0);
5150 format %{ %}
5151 interface(REG_INTER);
5152 %}
5153
5154 operand vRegD_V6()
5155 %{
5156 constraint(ALLOC_IN_RC(v6_reg));
5157 match(RegD);
5158 op_cost(0);
5159 format %{ %}
5160 interface(REG_INTER);
5161 %}
5162
5163 operand vRegD_V7()
5164 %{
5165 constraint(ALLOC_IN_RC(v7_reg));
5166 match(RegD);
5167 op_cost(0);
5168 format %{ %}
5169 interface(REG_INTER);
5170 %}
5171
5172 operand vRegD_V12()
5173 %{
5174 constraint(ALLOC_IN_RC(v12_reg));
5175 match(RegD);
5176 op_cost(0);
5177 format %{ %}
5178 interface(REG_INTER);
5179 %}
5180
5181 operand vRegD_V13()
5182 %{
5183 constraint(ALLOC_IN_RC(v13_reg));
5184 match(RegD);
5185 op_cost(0);
5186 format %{ %}
5187 interface(REG_INTER);
5188 %}
5189
5190 operand pReg()
5191 %{
5192 constraint(ALLOC_IN_RC(pr_reg));
5193 match(RegVectMask);
5194 match(pRegGov);
5195 op_cost(0);
5196 format %{ %}
5197 interface(REG_INTER);
5198 %}
5199
5200 operand pRegGov()
5201 %{
5202 constraint(ALLOC_IN_RC(gov_pr));
5203 match(RegVectMask);
5204 match(pReg);
5205 op_cost(0);
5206 format %{ %}
5207 interface(REG_INTER);
5208 %}
5209
5210 operand pRegGov_P0()
5211 %{
5212 constraint(ALLOC_IN_RC(p0_reg));
5213 match(RegVectMask);
5214 op_cost(0);
5215 format %{ %}
5216 interface(REG_INTER);
5217 %}
5218
5219 operand pRegGov_P1()
5220 %{
5221 constraint(ALLOC_IN_RC(p1_reg));
5222 match(RegVectMask);
5223 op_cost(0);
5224 format %{ %}
5225 interface(REG_INTER);
5226 %}
5227
5228 // Flags register, used as output of signed compare instructions
5229
5230 // note that on AArch64 we also use this register as the output for
5231 // for floating point compare instructions (CmpF CmpD). this ensures
5232 // that ordered inequality tests use GT, GE, LT or LE none of which
5233 // pass through cases where the result is unordered i.e. one or both
5234 // inputs to the compare is a NaN. this means that the ideal code can
5235 // replace e.g. a GT with an LE and not end up capturing the NaN case
5236 // (where the comparison should always fail). EQ and NE tests are
5237 // always generated in ideal code so that unordered folds into the NE
5238 // case, matching the behaviour of AArch64 NE.
5239 //
5240 // This differs from x86 where the outputs of FP compares use a
5241 // special FP flags registers and where compares based on this
5242 // register are distinguished into ordered inequalities (cmpOpUCF) and
5243 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5244 // to explicitly handle the unordered case in branches. x86 also has
5245 // to include extra CMoveX rules to accept a cmpOpUCF input.
5246
5247 operand rFlagsReg()
5248 %{
5249 constraint(ALLOC_IN_RC(int_flags));
5250 match(RegFlags);
5251
5252 op_cost(0);
5253 format %{ "RFLAGS" %}
5254 interface(REG_INTER);
5255 %}
5256
5257 // Flags register, used as output of unsigned compare instructions
5258 operand rFlagsRegU()
5259 %{
5260 constraint(ALLOC_IN_RC(int_flags));
5261 match(RegFlags);
5262
5263 op_cost(0);
5264 format %{ "RFLAGSU" %}
5265 interface(REG_INTER);
5266 %}
5267
5268 // Special Registers
5269
5270 // Method Register
5271 operand inline_cache_RegP(iRegP reg)
5272 %{
5273 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5274 match(reg);
5275 match(iRegPNoSp);
5276 op_cost(0);
5277 format %{ %}
5278 interface(REG_INTER);
5279 %}
5280
5281 // Thread Register
5282 operand thread_RegP(iRegP reg)
5283 %{
5284 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5285 match(reg);
5286 op_cost(0);
5287 format %{ %}
5288 interface(REG_INTER);
5289 %}
5290
5291 //----------Memory Operands----------------------------------------------------
5292
5293 operand indirect(iRegP reg)
5294 %{
5295 constraint(ALLOC_IN_RC(ptr_reg));
5296 match(reg);
5297 op_cost(0);
5298 format %{ "[$reg]" %}
5299 interface(MEMORY_INTER) %{
5300 base($reg);
5301 index(0xffffffff);
5302 scale(0x0);
5303 disp(0x0);
5304 %}
5305 %}
5306
5307 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5308 %{
5309 constraint(ALLOC_IN_RC(ptr_reg));
5310 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5311 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5312 op_cost(0);
5313 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5314 interface(MEMORY_INTER) %{
5315 base($reg);
5316 index($ireg);
5317 scale($scale);
5318 disp(0x0);
5319 %}
5320 %}
5321
5322 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5323 %{
5324 constraint(ALLOC_IN_RC(ptr_reg));
5325 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5326 match(AddP reg (LShiftL lreg scale));
5327 op_cost(0);
5328 format %{ "$reg, $lreg lsl($scale)" %}
5329 interface(MEMORY_INTER) %{
5330 base($reg);
5331 index($lreg);
5332 scale($scale);
5333 disp(0x0);
5334 %}
5335 %}
5336
5337 operand indIndexI2L(iRegP reg, iRegI ireg)
5338 %{
5339 constraint(ALLOC_IN_RC(ptr_reg));
5340 match(AddP reg (ConvI2L ireg));
5341 op_cost(0);
5342 format %{ "$reg, $ireg, 0, I2L" %}
5343 interface(MEMORY_INTER) %{
5344 base($reg);
5345 index($ireg);
5346 scale(0x0);
5347 disp(0x0);
5348 %}
5349 %}
5350
5351 operand indIndex(iRegP reg, iRegL lreg)
5352 %{
5353 constraint(ALLOC_IN_RC(ptr_reg));
5354 match(AddP reg lreg);
5355 op_cost(0);
5356 format %{ "$reg, $lreg" %}
5357 interface(MEMORY_INTER) %{
5358 base($reg);
5359 index($lreg);
5360 scale(0x0);
5361 disp(0x0);
5362 %}
5363 %}
5364
5365 operand indOffI1(iRegP reg, immIOffset1 off)
5366 %{
5367 constraint(ALLOC_IN_RC(ptr_reg));
5368 match(AddP reg off);
5369 op_cost(0);
5370 format %{ "[$reg, $off]" %}
5371 interface(MEMORY_INTER) %{
5372 base($reg);
5373 index(0xffffffff);
5374 scale(0x0);
5375 disp($off);
5376 %}
5377 %}
5378
5379 operand indOffI2(iRegP reg, immIOffset2 off)
5380 %{
5381 constraint(ALLOC_IN_RC(ptr_reg));
5382 match(AddP reg off);
5383 op_cost(0);
5384 format %{ "[$reg, $off]" %}
5385 interface(MEMORY_INTER) %{
5386 base($reg);
5387 index(0xffffffff);
5388 scale(0x0);
5389 disp($off);
5390 %}
5391 %}
5392
5393 operand indOffI4(iRegP reg, immIOffset4 off)
5394 %{
5395 constraint(ALLOC_IN_RC(ptr_reg));
5396 match(AddP reg off);
5397 op_cost(0);
5398 format %{ "[$reg, $off]" %}
5399 interface(MEMORY_INTER) %{
5400 base($reg);
5401 index(0xffffffff);
5402 scale(0x0);
5403 disp($off);
5404 %}
5405 %}
5406
5407 operand indOffI8(iRegP reg, immIOffset8 off)
5408 %{
5409 constraint(ALLOC_IN_RC(ptr_reg));
5410 match(AddP reg off);
5411 op_cost(0);
5412 format %{ "[$reg, $off]" %}
5413 interface(MEMORY_INTER) %{
5414 base($reg);
5415 index(0xffffffff);
5416 scale(0x0);
5417 disp($off);
5418 %}
5419 %}
5420
5421 operand indOffI16(iRegP reg, immIOffset16 off)
5422 %{
5423 constraint(ALLOC_IN_RC(ptr_reg));
5424 match(AddP reg off);
5425 op_cost(0);
5426 format %{ "[$reg, $off]" %}
5427 interface(MEMORY_INTER) %{
5428 base($reg);
5429 index(0xffffffff);
5430 scale(0x0);
5431 disp($off);
5432 %}
5433 %}
5434
5435 operand indOffL1(iRegP reg, immLoffset1 off)
5436 %{
5437 constraint(ALLOC_IN_RC(ptr_reg));
5438 match(AddP reg off);
5439 op_cost(0);
5440 format %{ "[$reg, $off]" %}
5441 interface(MEMORY_INTER) %{
5442 base($reg);
5443 index(0xffffffff);
5444 scale(0x0);
5445 disp($off);
5446 %}
5447 %}
5448
5449 operand indOffL2(iRegP reg, immLoffset2 off)
5450 %{
5451 constraint(ALLOC_IN_RC(ptr_reg));
5452 match(AddP reg off);
5453 op_cost(0);
5454 format %{ "[$reg, $off]" %}
5455 interface(MEMORY_INTER) %{
5456 base($reg);
5457 index(0xffffffff);
5458 scale(0x0);
5459 disp($off);
5460 %}
5461 %}
5462
5463 operand indOffL4(iRegP reg, immLoffset4 off)
5464 %{
5465 constraint(ALLOC_IN_RC(ptr_reg));
5466 match(AddP reg off);
5467 op_cost(0);
5468 format %{ "[$reg, $off]" %}
5469 interface(MEMORY_INTER) %{
5470 base($reg);
5471 index(0xffffffff);
5472 scale(0x0);
5473 disp($off);
5474 %}
5475 %}
5476
5477 operand indOffL8(iRegP reg, immLoffset8 off)
5478 %{
5479 constraint(ALLOC_IN_RC(ptr_reg));
5480 match(AddP reg off);
5481 op_cost(0);
5482 format %{ "[$reg, $off]" %}
5483 interface(MEMORY_INTER) %{
5484 base($reg);
5485 index(0xffffffff);
5486 scale(0x0);
5487 disp($off);
5488 %}
5489 %}
5490
5491 operand indOffL16(iRegP reg, immLoffset16 off)
5492 %{
5493 constraint(ALLOC_IN_RC(ptr_reg));
5494 match(AddP reg off);
5495 op_cost(0);
5496 format %{ "[$reg, $off]" %}
5497 interface(MEMORY_INTER) %{
5498 base($reg);
5499 index(0xffffffff);
5500 scale(0x0);
5501 disp($off);
5502 %}
5503 %}
5504
5505 operand indirectX2P(iRegL reg)
5506 %{
5507 constraint(ALLOC_IN_RC(ptr_reg));
5508 match(CastX2P reg);
5509 op_cost(0);
5510 format %{ "[$reg]\t# long -> ptr" %}
5511 interface(MEMORY_INTER) %{
5512 base($reg);
5513 index(0xffffffff);
5514 scale(0x0);
5515 disp(0x0);
5516 %}
5517 %}
5518
5519 operand indOffX2P(iRegL reg, immLOffset off)
5520 %{
5521 constraint(ALLOC_IN_RC(ptr_reg));
5522 match(AddP (CastX2P reg) off);
5523 op_cost(0);
5524 format %{ "[$reg, $off]\t# long -> ptr" %}
5525 interface(MEMORY_INTER) %{
5526 base($reg);
5527 index(0xffffffff);
5528 scale(0x0);
5529 disp($off);
5530 %}
5531 %}
5532
5533 operand indirectN(iRegN reg)
5534 %{
5535 predicate(CompressedOops::shift() == 0);
5536 constraint(ALLOC_IN_RC(ptr_reg));
5537 match(DecodeN reg);
5538 op_cost(0);
5539 format %{ "[$reg]\t# narrow" %}
5540 interface(MEMORY_INTER) %{
5541 base($reg);
5542 index(0xffffffff);
5543 scale(0x0);
5544 disp(0x0);
5545 %}
5546 %}
5547
5548 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5549 %{
5550 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5551 constraint(ALLOC_IN_RC(ptr_reg));
5552 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5553 op_cost(0);
5554 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5555 interface(MEMORY_INTER) %{
5556 base($reg);
5557 index($ireg);
5558 scale($scale);
5559 disp(0x0);
5560 %}
5561 %}
5562
5563 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5564 %{
5565 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5566 constraint(ALLOC_IN_RC(ptr_reg));
5567 match(AddP (DecodeN reg) (LShiftL lreg scale));
5568 op_cost(0);
5569 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5570 interface(MEMORY_INTER) %{
5571 base($reg);
5572 index($lreg);
5573 scale($scale);
5574 disp(0x0);
5575 %}
5576 %}
5577
5578 operand indIndexI2LN(iRegN reg, iRegI ireg)
5579 %{
5580 predicate(CompressedOops::shift() == 0);
5581 constraint(ALLOC_IN_RC(ptr_reg));
5582 match(AddP (DecodeN reg) (ConvI2L ireg));
5583 op_cost(0);
5584 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5585 interface(MEMORY_INTER) %{
5586 base($reg);
5587 index($ireg);
5588 scale(0x0);
5589 disp(0x0);
5590 %}
5591 %}
5592
5593 operand indIndexN(iRegN reg, iRegL lreg)
5594 %{
5595 predicate(CompressedOops::shift() == 0);
5596 constraint(ALLOC_IN_RC(ptr_reg));
5597 match(AddP (DecodeN reg) lreg);
5598 op_cost(0);
5599 format %{ "$reg, $lreg\t# narrow" %}
5600 interface(MEMORY_INTER) %{
5601 base($reg);
5602 index($lreg);
5603 scale(0x0);
5604 disp(0x0);
5605 %}
5606 %}
5607
5608 operand indOffIN(iRegN reg, immIOffset off)
5609 %{
5610 predicate(CompressedOops::shift() == 0);
5611 constraint(ALLOC_IN_RC(ptr_reg));
5612 match(AddP (DecodeN reg) off);
5613 op_cost(0);
5614 format %{ "[$reg, $off]\t# narrow" %}
5615 interface(MEMORY_INTER) %{
5616 base($reg);
5617 index(0xffffffff);
5618 scale(0x0);
5619 disp($off);
5620 %}
5621 %}
5622
5623 operand indOffLN(iRegN reg, immLOffset off)
5624 %{
5625 predicate(CompressedOops::shift() == 0);
5626 constraint(ALLOC_IN_RC(ptr_reg));
5627 match(AddP (DecodeN reg) off);
5628 op_cost(0);
5629 format %{ "[$reg, $off]\t# narrow" %}
5630 interface(MEMORY_INTER) %{
5631 base($reg);
5632 index(0xffffffff);
5633 scale(0x0);
5634 disp($off);
5635 %}
5636 %}
5637
5638
5639 //----------Special Memory Operands--------------------------------------------
5640 // Stack Slot Operand - This operand is used for loading and storing temporary
5641 // values on the stack where a match requires a value to
5642 // flow through memory.
5643 operand stackSlotP(sRegP reg)
5644 %{
5645 constraint(ALLOC_IN_RC(stack_slots));
5646 op_cost(100);
5647 // No match rule because this operand is only generated in matching
5648 // match(RegP);
5649 format %{ "[$reg]" %}
5650 interface(MEMORY_INTER) %{
5651 base(0x1e); // RSP
5652 index(0x0); // No Index
5653 scale(0x0); // No Scale
5654 disp($reg); // Stack Offset
5655 %}
5656 %}
5657
5658 operand stackSlotI(sRegI reg)
5659 %{
5660 constraint(ALLOC_IN_RC(stack_slots));
5661 // No match rule because this operand is only generated in matching
5662 // match(RegI);
5663 format %{ "[$reg]" %}
5664 interface(MEMORY_INTER) %{
5665 base(0x1e); // RSP
5666 index(0x0); // No Index
5667 scale(0x0); // No Scale
5668 disp($reg); // Stack Offset
5669 %}
5670 %}
5671
5672 operand stackSlotF(sRegF reg)
5673 %{
5674 constraint(ALLOC_IN_RC(stack_slots));
5675 // No match rule because this operand is only generated in matching
5676 // match(RegF);
5677 format %{ "[$reg]" %}
5678 interface(MEMORY_INTER) %{
5679 base(0x1e); // RSP
5680 index(0x0); // No Index
5681 scale(0x0); // No Scale
5682 disp($reg); // Stack Offset
5683 %}
5684 %}
5685
5686 operand stackSlotD(sRegD reg)
5687 %{
5688 constraint(ALLOC_IN_RC(stack_slots));
5689 // No match rule because this operand is only generated in matching
5690 // match(RegD);
5691 format %{ "[$reg]" %}
5692 interface(MEMORY_INTER) %{
5693 base(0x1e); // RSP
5694 index(0x0); // No Index
5695 scale(0x0); // No Scale
5696 disp($reg); // Stack Offset
5697 %}
5698 %}
5699
5700 operand stackSlotL(sRegL reg)
5701 %{
5702 constraint(ALLOC_IN_RC(stack_slots));
5703 // No match rule because this operand is only generated in matching
5704 // match(RegL);
5705 format %{ "[$reg]" %}
5706 interface(MEMORY_INTER) %{
5707 base(0x1e); // RSP
5708 index(0x0); // No Index
5709 scale(0x0); // No Scale
5710 disp($reg); // Stack Offset
5711 %}
5712 %}
5713
5714 // Operands for expressing Control Flow
5715 // NOTE: Label is a predefined operand which should not be redefined in
5716 // the AD file. It is generically handled within the ADLC.
5717
5718 //----------Conditional Branch Operands----------------------------------------
5719 // Comparison Op - This is the operation of the comparison, and is limited to
5720 // the following set of codes:
5721 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5722 //
5723 // Other attributes of the comparison, such as unsignedness, are specified
5724 // by the comparison instruction that sets a condition code flags register.
5725 // That result is represented by a flags operand whose subtype is appropriate
5726 // to the unsignedness (etc.) of the comparison.
5727 //
5728 // Later, the instruction which matches both the Comparison Op (a Bool) and
5729 // the flags (produced by the Cmp) specifies the coding of the comparison op
5730 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5731
5732 // used for signed integral comparisons and fp comparisons
5733
5734 operand cmpOp()
5735 %{
5736 match(Bool);
5737
5738 format %{ "" %}
5739 interface(COND_INTER) %{
5740 equal(0x0, "eq");
5741 not_equal(0x1, "ne");
5742 less(0xb, "lt");
5743 greater_equal(0xa, "ge");
5744 less_equal(0xd, "le");
5745 greater(0xc, "gt");
5746 overflow(0x6, "vs");
5747 no_overflow(0x7, "vc");
5748 %}
5749 %}
5750
5751 // used for unsigned integral comparisons
5752
5753 operand cmpOpU()
5754 %{
5755 match(Bool);
5756
5757 format %{ "" %}
5758 interface(COND_INTER) %{
5759 equal(0x0, "eq");
5760 not_equal(0x1, "ne");
5761 less(0x3, "lo");
5762 greater_equal(0x2, "hs");
5763 less_equal(0x9, "ls");
5764 greater(0x8, "hi");
5765 overflow(0x6, "vs");
5766 no_overflow(0x7, "vc");
5767 %}
5768 %}
5769
5770 // used for certain integral comparisons which can be
5771 // converted to cbxx or tbxx instructions
5772
5773 operand cmpOpEqNe()
5774 %{
5775 match(Bool);
5776 op_cost(0);
5777 predicate(n->as_Bool()->_test._test == BoolTest::ne
5778 || n->as_Bool()->_test._test == BoolTest::eq);
5779
5780 format %{ "" %}
5781 interface(COND_INTER) %{
5782 equal(0x0, "eq");
5783 not_equal(0x1, "ne");
5784 less(0xb, "lt");
5785 greater_equal(0xa, "ge");
5786 less_equal(0xd, "le");
5787 greater(0xc, "gt");
5788 overflow(0x6, "vs");
5789 no_overflow(0x7, "vc");
5790 %}
5791 %}
5792
5793 // used for certain integral comparisons which can be
5794 // converted to cbxx or tbxx instructions
5795
5796 operand cmpOpLtGe()
5797 %{
5798 match(Bool);
5799 op_cost(0);
5800
5801 predicate(n->as_Bool()->_test._test == BoolTest::lt
5802 || n->as_Bool()->_test._test == BoolTest::ge);
5803
5804 format %{ "" %}
5805 interface(COND_INTER) %{
5806 equal(0x0, "eq");
5807 not_equal(0x1, "ne");
5808 less(0xb, "lt");
5809 greater_equal(0xa, "ge");
5810 less_equal(0xd, "le");
5811 greater(0xc, "gt");
5812 overflow(0x6, "vs");
5813 no_overflow(0x7, "vc");
5814 %}
5815 %}
5816
5817 // used for certain unsigned integral comparisons which can be
5818 // converted to cbxx or tbxx instructions
5819
5820 operand cmpOpUEqNeLeGt()
5821 %{
5822 match(Bool);
5823 op_cost(0);
5824
5825 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5826 n->as_Bool()->_test._test == BoolTest::ne ||
5827 n->as_Bool()->_test._test == BoolTest::le ||
5828 n->as_Bool()->_test._test == BoolTest::gt);
5829
5830 format %{ "" %}
5831 interface(COND_INTER) %{
5832 equal(0x0, "eq");
5833 not_equal(0x1, "ne");
5834 less(0x3, "lo");
5835 greater_equal(0x2, "hs");
5836 less_equal(0x9, "ls");
5837 greater(0x8, "hi");
5838 overflow(0x6, "vs");
5839 no_overflow(0x7, "vc");
5840 %}
5841 %}
5842
5843 // Special operand allowing long args to int ops to be truncated for free
5844
5845 operand iRegL2I(iRegL reg) %{
5846
5847 op_cost(0);
5848
5849 match(ConvL2I reg);
5850
5851 format %{ "l2i($reg)" %}
5852
5853 interface(REG_INTER)
5854 %}
5855
5856 operand iRegL2P(iRegL reg) %{
5857
5858 op_cost(0);
5859
5860 match(CastX2P reg);
5861
5862 format %{ "l2p($reg)" %}
5863
5864 interface(REG_INTER)
5865 %}
5866
5867 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5868 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5869 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5870 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5871
5872 //----------OPERAND CLASSES----------------------------------------------------
5873 // Operand Classes are groups of operands that are used as to simplify
5874 // instruction definitions by not requiring the AD writer to specify
5875 // separate instructions for every form of operand when the
5876 // instruction accepts multiple operand types with the same basic
5877 // encoding and format. The classic case of this is memory operands.
5878
5879 // memory is used to define read/write location for load/store
5880 // instruction defs. we can turn a memory op into an Address
5881
5882 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5883 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5884
5885 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5886 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5887
5888 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5889 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5890
5891 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5892 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5893
5894 // All of the memory operands. For the pipeline description.
5895 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5896 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5897 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5898
5899
5900 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5901 // operations. it allows the src to be either an iRegI or a (ConvL2I
5902 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5903 // can be elided because the 32-bit instruction will just employ the
5904 // lower 32 bits anyway.
5905 //
5906 // n.b. this does not elide all L2I conversions. if the truncated
5907 // value is consumed by more than one operation then the ConvL2I
5908 // cannot be bundled into the consuming nodes so an l2i gets planted
5909 // (actually a movw $dst $src) and the downstream instructions consume
5910 // the result of the l2i as an iRegI input. That's a shame since the
5911 // movw is actually redundant but its not too costly.
5912
5913 opclass iRegIorL2I(iRegI, iRegL2I);
5914 opclass iRegPorL2P(iRegP, iRegL2P);
5915
5916 //----------PIPELINE-----------------------------------------------------------
5917 // Rules which define the behavior of the target architectures pipeline.
5918
5919 // For specific pipelines, eg A53, define the stages of that pipeline
5920 //pipe_desc(ISS, EX1, EX2, WR);
5921 #define ISS S0
5922 #define EX1 S1
5923 #define EX2 S2
5924 #define WR S3
5925
5926 // Integer ALU reg operation
5927 pipeline %{
5928
5929 attributes %{
5930 // ARM instructions are of fixed length
5931 fixed_size_instructions; // Fixed size instructions TODO does
5932 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5933 // ARM instructions come in 32-bit word units
5934 instruction_unit_size = 4; // An instruction is 4 bytes long
5935 instruction_fetch_unit_size = 64; // The processor fetches one line
5936 instruction_fetch_units = 1; // of 64 bytes
5937 %}
5938
5939 // We don't use an actual pipeline model so don't care about resources
5940 // or description. we do use pipeline classes to introduce fixed
5941 // latencies
5942
5943 //----------RESOURCES----------------------------------------------------------
5944 // Resources are the functional units available to the machine
5945
5946 resources( INS0, INS1, INS01 = INS0 | INS1,
5947 ALU0, ALU1, ALU = ALU0 | ALU1,
5948 MAC,
5949 DIV,
5950 BRANCH,
5951 LDST,
5952 NEON_FP);
5953
5954 //----------PIPELINE DESCRIPTION-----------------------------------------------
5955 // Pipeline Description specifies the stages in the machine's pipeline
5956
5957 // Define the pipeline as a generic 6 stage pipeline
5958 pipe_desc(S0, S1, S2, S3, S4, S5);
5959
5960 //----------PIPELINE CLASSES---------------------------------------------------
5961 // Pipeline Classes describe the stages in which input and output are
5962 // referenced by the hardware pipeline.
5963
5964 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5965 %{
5966 single_instruction;
5967 src1 : S1(read);
5968 src2 : S2(read);
5969 dst : S5(write);
5970 INS01 : ISS;
5971 NEON_FP : S5;
5972 %}
5973
5974 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5975 %{
5976 single_instruction;
5977 src1 : S1(read);
5978 src2 : S2(read);
5979 dst : S5(write);
5980 INS01 : ISS;
5981 NEON_FP : S5;
5982 %}
5983
5984 pipe_class fp_uop_s(vRegF dst, vRegF src)
5985 %{
5986 single_instruction;
5987 src : S1(read);
5988 dst : S5(write);
5989 INS01 : ISS;
5990 NEON_FP : S5;
5991 %}
5992
5993 pipe_class fp_uop_d(vRegD dst, vRegD src)
5994 %{
5995 single_instruction;
5996 src : S1(read);
5997 dst : S5(write);
5998 INS01 : ISS;
5999 NEON_FP : S5;
6000 %}
6001
6002 pipe_class fp_d2f(vRegF dst, vRegD src)
6003 %{
6004 single_instruction;
6005 src : S1(read);
6006 dst : S5(write);
6007 INS01 : ISS;
6008 NEON_FP : S5;
6009 %}
6010
6011 pipe_class fp_f2d(vRegD dst, vRegF src)
6012 %{
6013 single_instruction;
6014 src : S1(read);
6015 dst : S5(write);
6016 INS01 : ISS;
6017 NEON_FP : S5;
6018 %}
6019
6020 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6021 %{
6022 single_instruction;
6023 src : S1(read);
6024 dst : S5(write);
6025 INS01 : ISS;
6026 NEON_FP : S5;
6027 %}
6028
6029 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6030 %{
6031 single_instruction;
6032 src : S1(read);
6033 dst : S5(write);
6034 INS01 : ISS;
6035 NEON_FP : S5;
6036 %}
6037
6038 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6039 %{
6040 single_instruction;
6041 src : S1(read);
6042 dst : S5(write);
6043 INS01 : ISS;
6044 NEON_FP : S5;
6045 %}
6046
6047 pipe_class fp_l2f(vRegF dst, iRegL src)
6048 %{
6049 single_instruction;
6050 src : S1(read);
6051 dst : S5(write);
6052 INS01 : ISS;
6053 NEON_FP : S5;
6054 %}
6055
6056 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6057 %{
6058 single_instruction;
6059 src : S1(read);
6060 dst : S5(write);
6061 INS01 : ISS;
6062 NEON_FP : S5;
6063 %}
6064
6065 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6066 %{
6067 single_instruction;
6068 src : S1(read);
6069 dst : S5(write);
6070 INS01 : ISS;
6071 NEON_FP : S5;
6072 %}
6073
6074 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6075 %{
6076 single_instruction;
6077 src : S1(read);
6078 dst : S5(write);
6079 INS01 : ISS;
6080 NEON_FP : S5;
6081 %}
6082
6083 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6084 %{
6085 single_instruction;
6086 src : S1(read);
6087 dst : S5(write);
6088 INS01 : ISS;
6089 NEON_FP : S5;
6090 %}
6091
6092 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6093 %{
6094 single_instruction;
6095 src1 : S1(read);
6096 src2 : S2(read);
6097 dst : S5(write);
6098 INS0 : ISS;
6099 NEON_FP : S5;
6100 %}
6101
6102 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6103 %{
6104 single_instruction;
6105 src1 : S1(read);
6106 src2 : S2(read);
6107 dst : S5(write);
6108 INS0 : ISS;
6109 NEON_FP : S5;
6110 %}
6111
6112 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6113 %{
6114 single_instruction;
6115 cr : S1(read);
6116 src1 : S1(read);
6117 src2 : S1(read);
6118 dst : S3(write);
6119 INS01 : ISS;
6120 NEON_FP : S3;
6121 %}
6122
6123 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6124 %{
6125 single_instruction;
6126 cr : S1(read);
6127 src1 : S1(read);
6128 src2 : S1(read);
6129 dst : S3(write);
6130 INS01 : ISS;
6131 NEON_FP : S3;
6132 %}
6133
6134 pipe_class fp_imm_s(vRegF dst)
6135 %{
6136 single_instruction;
6137 dst : S3(write);
6138 INS01 : ISS;
6139 NEON_FP : S3;
6140 %}
6141
6142 pipe_class fp_imm_d(vRegD dst)
6143 %{
6144 single_instruction;
6145 dst : S3(write);
6146 INS01 : ISS;
6147 NEON_FP : S3;
6148 %}
6149
6150 pipe_class fp_load_constant_s(vRegF dst)
6151 %{
6152 single_instruction;
6153 dst : S4(write);
6154 INS01 : ISS;
6155 NEON_FP : S4;
6156 %}
6157
6158 pipe_class fp_load_constant_d(vRegD dst)
6159 %{
6160 single_instruction;
6161 dst : S4(write);
6162 INS01 : ISS;
6163 NEON_FP : S4;
6164 %}
6165
6166 //------- Integer ALU operations --------------------------
6167
6168 // Integer ALU reg-reg operation
6169 // Operands needed in EX1, result generated in EX2
6170 // Eg. ADD x0, x1, x2
6171 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6172 %{
6173 single_instruction;
6174 dst : EX2(write);
6175 src1 : EX1(read);
6176 src2 : EX1(read);
6177 INS01 : ISS; // Dual issue as instruction 0 or 1
6178 ALU : EX2;
6179 %}
6180
6181 // Integer ALU reg-reg operation with constant shift
6182 // Shifted register must be available in LATE_ISS instead of EX1
6183 // Eg. ADD x0, x1, x2, LSL #2
6184 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6185 %{
6186 single_instruction;
6187 dst : EX2(write);
6188 src1 : EX1(read);
6189 src2 : ISS(read);
6190 INS01 : ISS;
6191 ALU : EX2;
6192 %}
6193
6194 // Integer ALU reg operation with constant shift
6195 // Eg. LSL x0, x1, #shift
6196 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6197 %{
6198 single_instruction;
6199 dst : EX2(write);
6200 src1 : ISS(read);
6201 INS01 : ISS;
6202 ALU : EX2;
6203 %}
6204
6205 // Integer ALU reg-reg operation with variable shift
6206 // Both operands must be available in LATE_ISS instead of EX1
6207 // Result is available in EX1 instead of EX2
6208 // Eg. LSLV x0, x1, x2
6209 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6210 %{
6211 single_instruction;
6212 dst : EX1(write);
6213 src1 : ISS(read);
6214 src2 : ISS(read);
6215 INS01 : ISS;
6216 ALU : EX1;
6217 %}
6218
6219 // Integer ALU reg-reg operation with extract
6220 // As for _vshift above, but result generated in EX2
6221 // Eg. EXTR x0, x1, x2, #N
6222 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6223 %{
6224 single_instruction;
6225 dst : EX2(write);
6226 src1 : ISS(read);
6227 src2 : ISS(read);
6228 INS1 : ISS; // Can only dual issue as Instruction 1
6229 ALU : EX1;
6230 %}
6231
6232 // Integer ALU reg operation
6233 // Eg. NEG x0, x1
6234 pipe_class ialu_reg(iRegI dst, iRegI src)
6235 %{
6236 single_instruction;
6237 dst : EX2(write);
6238 src : EX1(read);
6239 INS01 : ISS;
6240 ALU : EX2;
6241 %}
6242
6243 // Integer ALU reg mmediate operation
6244 // Eg. ADD x0, x1, #N
6245 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6246 %{
6247 single_instruction;
6248 dst : EX2(write);
6249 src1 : EX1(read);
6250 INS01 : ISS;
6251 ALU : EX2;
6252 %}
6253
6254 // Integer ALU immediate operation (no source operands)
6255 // Eg. MOV x0, #N
6256 pipe_class ialu_imm(iRegI dst)
6257 %{
6258 single_instruction;
6259 dst : EX1(write);
6260 INS01 : ISS;
6261 ALU : EX1;
6262 %}
6263
6264 //------- Compare operation -------------------------------
6265
6266 // Compare reg-reg
6267 // Eg. CMP x0, x1
6268 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6269 %{
6270 single_instruction;
6271 // fixed_latency(16);
6272 cr : EX2(write);
6273 op1 : EX1(read);
6274 op2 : EX1(read);
6275 INS01 : ISS;
6276 ALU : EX2;
6277 %}
6278
6279 // Compare reg-reg
6280 // Eg. CMP x0, #N
6281 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6282 %{
6283 single_instruction;
6284 // fixed_latency(16);
6285 cr : EX2(write);
6286 op1 : EX1(read);
6287 INS01 : ISS;
6288 ALU : EX2;
6289 %}
6290
6291 //------- Conditional instructions ------------------------
6292
6293 // Conditional no operands
6294 // Eg. CSINC x0, zr, zr, <cond>
6295 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6296 %{
6297 single_instruction;
6298 cr : EX1(read);
6299 dst : EX2(write);
6300 INS01 : ISS;
6301 ALU : EX2;
6302 %}
6303
6304 // Conditional 2 operand
6305 // EG. CSEL X0, X1, X2, <cond>
6306 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6307 %{
6308 single_instruction;
6309 cr : EX1(read);
6310 src1 : EX1(read);
6311 src2 : EX1(read);
6312 dst : EX2(write);
6313 INS01 : ISS;
6314 ALU : EX2;
6315 %}
6316
6317 // Conditional 2 operand
6318 // EG. CSEL X0, X1, X2, <cond>
6319 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6320 %{
6321 single_instruction;
6322 cr : EX1(read);
6323 src : EX1(read);
6324 dst : EX2(write);
6325 INS01 : ISS;
6326 ALU : EX2;
6327 %}
6328
6329 //------- Multiply pipeline operations --------------------
6330
6331 // Multiply reg-reg
6332 // Eg. MUL w0, w1, w2
6333 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6334 %{
6335 single_instruction;
6336 dst : WR(write);
6337 src1 : ISS(read);
6338 src2 : ISS(read);
6339 INS01 : ISS;
6340 MAC : WR;
6341 %}
6342
6343 // Multiply accumulate
6344 // Eg. MADD w0, w1, w2, w3
6345 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6346 %{
6347 single_instruction;
6348 dst : WR(write);
6349 src1 : ISS(read);
6350 src2 : ISS(read);
6351 src3 : ISS(read);
6352 INS01 : ISS;
6353 MAC : WR;
6354 %}
6355
6356 // Eg. MUL w0, w1, w2
6357 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6358 %{
6359 single_instruction;
6360 fixed_latency(3); // Maximum latency for 64 bit mul
6361 dst : WR(write);
6362 src1 : ISS(read);
6363 src2 : ISS(read);
6364 INS01 : ISS;
6365 MAC : WR;
6366 %}
6367
6368 // Multiply accumulate
6369 // Eg. MADD w0, w1, w2, w3
6370 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6371 %{
6372 single_instruction;
6373 fixed_latency(3); // Maximum latency for 64 bit mul
6374 dst : WR(write);
6375 src1 : ISS(read);
6376 src2 : ISS(read);
6377 src3 : ISS(read);
6378 INS01 : ISS;
6379 MAC : WR;
6380 %}
6381
6382 //------- Divide pipeline operations --------------------
6383
6384 // Eg. SDIV w0, w1, w2
6385 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6386 %{
6387 single_instruction;
6388 fixed_latency(8); // Maximum latency for 32 bit divide
6389 dst : WR(write);
6390 src1 : ISS(read);
6391 src2 : ISS(read);
6392 INS0 : ISS; // Can only dual issue as instruction 0
6393 DIV : WR;
6394 %}
6395
6396 // Eg. SDIV x0, x1, x2
6397 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6398 %{
6399 single_instruction;
6400 fixed_latency(16); // Maximum latency for 64 bit divide
6401 dst : WR(write);
6402 src1 : ISS(read);
6403 src2 : ISS(read);
6404 INS0 : ISS; // Can only dual issue as instruction 0
6405 DIV : WR;
6406 %}
6407
6408 //------- Load pipeline operations ------------------------
6409
6410 // Load - prefetch
6411 // Eg. PFRM <mem>
6412 pipe_class iload_prefetch(memory mem)
6413 %{
6414 single_instruction;
6415 mem : ISS(read);
6416 INS01 : ISS;
6417 LDST : WR;
6418 %}
6419
6420 // Load - reg, mem
6421 // Eg. LDR x0, <mem>
6422 pipe_class iload_reg_mem(iRegI dst, memory mem)
6423 %{
6424 single_instruction;
6425 dst : WR(write);
6426 mem : ISS(read);
6427 INS01 : ISS;
6428 LDST : WR;
6429 %}
6430
6431 // Load - reg, reg
6432 // Eg. LDR x0, [sp, x1]
6433 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6434 %{
6435 single_instruction;
6436 dst : WR(write);
6437 src : ISS(read);
6438 INS01 : ISS;
6439 LDST : WR;
6440 %}
6441
6442 //------- Store pipeline operations -----------------------
6443
6444 // Store - zr, mem
6445 // Eg. STR zr, <mem>
6446 pipe_class istore_mem(memory mem)
6447 %{
6448 single_instruction;
6449 mem : ISS(read);
6450 INS01 : ISS;
6451 LDST : WR;
6452 %}
6453
6454 // Store - reg, mem
6455 // Eg. STR x0, <mem>
6456 pipe_class istore_reg_mem(iRegI src, memory mem)
6457 %{
6458 single_instruction;
6459 mem : ISS(read);
6460 src : EX2(read);
6461 INS01 : ISS;
6462 LDST : WR;
6463 %}
6464
6465 // Store - reg, reg
6466 // Eg. STR x0, [sp, x1]
6467 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6468 %{
6469 single_instruction;
6470 dst : ISS(read);
6471 src : EX2(read);
6472 INS01 : ISS;
6473 LDST : WR;
6474 %}
6475
6476 //------- Store pipeline operations -----------------------
6477
6478 // Branch
6479 pipe_class pipe_branch()
6480 %{
6481 single_instruction;
6482 INS01 : ISS;
6483 BRANCH : EX1;
6484 %}
6485
6486 // Conditional branch
6487 pipe_class pipe_branch_cond(rFlagsReg cr)
6488 %{
6489 single_instruction;
6490 cr : EX1(read);
6491 INS01 : ISS;
6492 BRANCH : EX1;
6493 %}
6494
6495 // Compare & Branch
6496 // EG. CBZ/CBNZ
6497 pipe_class pipe_cmp_branch(iRegI op1)
6498 %{
6499 single_instruction;
6500 op1 : EX1(read);
6501 INS01 : ISS;
6502 BRANCH : EX1;
6503 %}
6504
6505 //------- Synchronisation operations ----------------------
6506
6507 // Any operation requiring serialization.
6508 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6509 pipe_class pipe_serial()
6510 %{
6511 single_instruction;
6512 force_serialization;
6513 fixed_latency(16);
6514 INS01 : ISS(2); // Cannot dual issue with any other instruction
6515 LDST : WR;
6516 %}
6517
6518 // Generic big/slow expanded idiom - also serialized
6519 pipe_class pipe_slow()
6520 %{
6521 instruction_count(10);
6522 multiple_bundles;
6523 force_serialization;
6524 fixed_latency(16);
6525 INS01 : ISS(2); // Cannot dual issue with any other instruction
6526 LDST : WR;
6527 %}
6528
6529 // Empty pipeline class
6530 pipe_class pipe_class_empty()
6531 %{
6532 single_instruction;
6533 fixed_latency(0);
6534 %}
6535
6536 // Default pipeline class.
6537 pipe_class pipe_class_default()
6538 %{
6539 single_instruction;
6540 fixed_latency(2);
6541 %}
6542
6543 // Pipeline class for compares.
6544 pipe_class pipe_class_compare()
6545 %{
6546 single_instruction;
6547 fixed_latency(16);
6548 %}
6549
6550 // Pipeline class for memory operations.
6551 pipe_class pipe_class_memory()
6552 %{
6553 single_instruction;
6554 fixed_latency(16);
6555 %}
6556
6557 // Pipeline class for call.
6558 pipe_class pipe_class_call()
6559 %{
6560 single_instruction;
6561 fixed_latency(100);
6562 %}
6563
6564 // Define the class for the Nop node.
6565 define %{
6566 MachNop = pipe_class_empty;
6567 %}
6568
6569 %}
6570 //----------INSTRUCTIONS-------------------------------------------------------
6571 //
6572 // match -- States which machine-independent subtree may be replaced
6573 // by this instruction.
6574 // ins_cost -- The estimated cost of this instruction is used by instruction
6575 // selection to identify a minimum cost tree of machine
6576 // instructions that matches a tree of machine-independent
6577 // instructions.
6578 // format -- A string providing the disassembly for this instruction.
6579 // The value of an instruction's operand may be inserted
6580 // by referring to it with a '$' prefix.
6581 // opcode -- Three instruction opcodes may be provided. These are referred
6582 // to within an encode class as $primary, $secondary, and $tertiary
6583 // rrspectively. The primary opcode is commonly used to
6584 // indicate the type of machine instruction, while secondary
6585 // and tertiary are often used for prefix options or addressing
6586 // modes.
6587 // ins_encode -- A list of encode classes with parameters. The encode class
6588 // name must have been defined in an 'enc_class' specification
6589 // in the encode section of the architecture description.
6590
6591 // ============================================================================
6592 // Memory (Load/Store) Instructions
6593
6594 // Load Instructions
6595
6596 // Load Byte (8 bit signed)
6597 instruct loadB(iRegINoSp dst, memory1 mem)
6598 %{
6599 match(Set dst (LoadB mem));
6600 predicate(!needs_acquiring_load(n));
6601
6602 ins_cost(4 * INSN_COST);
6603 format %{ "ldrsbw $dst, $mem\t# byte" %}
6604
6605 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6606
6607 ins_pipe(iload_reg_mem);
6608 %}
6609
6610 // Load Byte (8 bit signed) into long
6611 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6612 %{
6613 match(Set dst (ConvI2L (LoadB mem)));
6614 predicate(!needs_acquiring_load(n->in(1)));
6615
6616 ins_cost(4 * INSN_COST);
6617 format %{ "ldrsb $dst, $mem\t# byte" %}
6618
6619 ins_encode(aarch64_enc_ldrsb(dst, mem));
6620
6621 ins_pipe(iload_reg_mem);
6622 %}
6623
6624 // Load Byte (8 bit unsigned)
6625 instruct loadUB(iRegINoSp dst, memory1 mem)
6626 %{
6627 match(Set dst (LoadUB mem));
6628 predicate(!needs_acquiring_load(n));
6629
6630 ins_cost(4 * INSN_COST);
6631 format %{ "ldrbw $dst, $mem\t# byte" %}
6632
6633 ins_encode(aarch64_enc_ldrb(dst, mem));
6634
6635 ins_pipe(iload_reg_mem);
6636 %}
6637
6638 // Load Byte (8 bit unsigned) into long
6639 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6640 %{
6641 match(Set dst (ConvI2L (LoadUB mem)));
6642 predicate(!needs_acquiring_load(n->in(1)));
6643
6644 ins_cost(4 * INSN_COST);
6645 format %{ "ldrb $dst, $mem\t# byte" %}
6646
6647 ins_encode(aarch64_enc_ldrb(dst, mem));
6648
6649 ins_pipe(iload_reg_mem);
6650 %}
6651
6652 // Load Short (16 bit signed)
6653 instruct loadS(iRegINoSp dst, memory2 mem)
6654 %{
6655 match(Set dst (LoadS mem));
6656 predicate(!needs_acquiring_load(n));
6657
6658 ins_cost(4 * INSN_COST);
6659 format %{ "ldrshw $dst, $mem\t# short" %}
6660
6661 ins_encode(aarch64_enc_ldrshw(dst, mem));
6662
6663 ins_pipe(iload_reg_mem);
6664 %}
6665
6666 // Load Short (16 bit signed) into long
6667 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6668 %{
6669 match(Set dst (ConvI2L (LoadS mem)));
6670 predicate(!needs_acquiring_load(n->in(1)));
6671
6672 ins_cost(4 * INSN_COST);
6673 format %{ "ldrsh $dst, $mem\t# short" %}
6674
6675 ins_encode(aarch64_enc_ldrsh(dst, mem));
6676
6677 ins_pipe(iload_reg_mem);
6678 %}
6679
6680 // Load Char (16 bit unsigned)
6681 instruct loadUS(iRegINoSp dst, memory2 mem)
6682 %{
6683 match(Set dst (LoadUS mem));
6684 predicate(!needs_acquiring_load(n));
6685
6686 ins_cost(4 * INSN_COST);
6687 format %{ "ldrh $dst, $mem\t# short" %}
6688
6689 ins_encode(aarch64_enc_ldrh(dst, mem));
6690
6691 ins_pipe(iload_reg_mem);
6692 %}
6693
6694 // Load Short/Char (16 bit unsigned) into long
6695 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6696 %{
6697 match(Set dst (ConvI2L (LoadUS mem)));
6698 predicate(!needs_acquiring_load(n->in(1)));
6699
6700 ins_cost(4 * INSN_COST);
6701 format %{ "ldrh $dst, $mem\t# short" %}
6702
6703 ins_encode(aarch64_enc_ldrh(dst, mem));
6704
6705 ins_pipe(iload_reg_mem);
6706 %}
6707
6708 // Load Integer (32 bit signed)
6709 instruct loadI(iRegINoSp dst, memory4 mem)
6710 %{
6711 match(Set dst (LoadI mem));
6712 predicate(!needs_acquiring_load(n));
6713
6714 ins_cost(4 * INSN_COST);
6715 format %{ "ldrw $dst, $mem\t# int" %}
6716
6717 ins_encode(aarch64_enc_ldrw(dst, mem));
6718
6719 ins_pipe(iload_reg_mem);
6720 %}
6721
6722 // Load Integer (32 bit signed) into long
6723 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6724 %{
6725 match(Set dst (ConvI2L (LoadI mem)));
6726 predicate(!needs_acquiring_load(n->in(1)));
6727
6728 ins_cost(4 * INSN_COST);
6729 format %{ "ldrsw $dst, $mem\t# int" %}
6730
6731 ins_encode(aarch64_enc_ldrsw(dst, mem));
6732
6733 ins_pipe(iload_reg_mem);
6734 %}
6735
6736 // Load Integer (32 bit unsigned) into long
6737 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6738 %{
6739 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6740 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6741
6742 ins_cost(4 * INSN_COST);
6743 format %{ "ldrw $dst, $mem\t# int" %}
6744
6745 ins_encode(aarch64_enc_ldrw(dst, mem));
6746
6747 ins_pipe(iload_reg_mem);
6748 %}
6749
6750 // Load Long (64 bit signed)
6751 instruct loadL(iRegLNoSp dst, memory8 mem)
6752 %{
6753 match(Set dst (LoadL mem));
6754 predicate(!needs_acquiring_load(n));
6755
6756 ins_cost(4 * INSN_COST);
6757 format %{ "ldr $dst, $mem\t# int" %}
6758
6759 ins_encode(aarch64_enc_ldr(dst, mem));
6760
6761 ins_pipe(iload_reg_mem);
6762 %}
6763
6764 // Load Range
6765 instruct loadRange(iRegINoSp dst, memory4 mem)
6766 %{
6767 match(Set dst (LoadRange mem));
6768
6769 ins_cost(4 * INSN_COST);
6770 format %{ "ldrw $dst, $mem\t# range" %}
6771
6772 ins_encode(aarch64_enc_ldrw(dst, mem));
6773
6774 ins_pipe(iload_reg_mem);
6775 %}
6776
6777 // Load Pointer
6778 instruct loadP(iRegPNoSp dst, memory8 mem)
6779 %{
6780 match(Set dst (LoadP mem));
6781 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6782
6783 ins_cost(4 * INSN_COST);
6784 format %{ "ldr $dst, $mem\t# ptr" %}
6785
6786 ins_encode(aarch64_enc_ldr(dst, mem));
6787
6788 ins_pipe(iload_reg_mem);
6789 %}
6790
6791 // Load Compressed Pointer
6792 instruct loadN(iRegNNoSp dst, memory4 mem)
6793 %{
6794 match(Set dst (LoadN mem));
6795 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6796
6797 ins_cost(4 * INSN_COST);
6798 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6799
6800 ins_encode(aarch64_enc_ldrw(dst, mem));
6801
6802 ins_pipe(iload_reg_mem);
6803 %}
6804
6805 // Load Klass Pointer
6806 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6807 %{
6808 match(Set dst (LoadKlass mem));
6809 predicate(!needs_acquiring_load(n));
6810
6811 ins_cost(4 * INSN_COST);
6812 format %{ "ldr $dst, $mem\t# class" %}
6813
6814 ins_encode(aarch64_enc_ldr(dst, mem));
6815
6816 ins_pipe(iload_reg_mem);
6817 %}
6818
6819 // Load Narrow Klass Pointer
6820 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6821 %{
6822 match(Set dst (LoadNKlass mem));
6823 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6824
6825 ins_cost(4 * INSN_COST);
6826 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6827
6828 ins_encode(aarch64_enc_ldrw(dst, mem));
6829
6830 ins_pipe(iload_reg_mem);
6831 %}
6832
6833 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6834 %{
6835 match(Set dst (LoadNKlass mem));
6836 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6837
6838 ins_cost(4 * INSN_COST);
6839 format %{
6840 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6841 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6842 %}
6843 ins_encode %{
6844 // inlined aarch64_enc_ldrw
6845 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6846 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6847 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6848 %}
6849 ins_pipe(iload_reg_mem);
6850 %}
6851
6852 // Load Float
6853 instruct loadF(vRegF dst, memory4 mem)
6854 %{
6855 match(Set dst (LoadF mem));
6856 predicate(!needs_acquiring_load(n));
6857
6858 ins_cost(4 * INSN_COST);
6859 format %{ "ldrs $dst, $mem\t# float" %}
6860
6861 ins_encode( aarch64_enc_ldrs(dst, mem) );
6862
6863 ins_pipe(pipe_class_memory);
6864 %}
6865
6866 // Load Double
6867 instruct loadD(vRegD dst, memory8 mem)
6868 %{
6869 match(Set dst (LoadD mem));
6870 predicate(!needs_acquiring_load(n));
6871
6872 ins_cost(4 * INSN_COST);
6873 format %{ "ldrd $dst, $mem\t# double" %}
6874
6875 ins_encode( aarch64_enc_ldrd(dst, mem) );
6876
6877 ins_pipe(pipe_class_memory);
6878 %}
6879
6880
6881 // Load Int Constant
6882 instruct loadConI(iRegINoSp dst, immI src)
6883 %{
6884 match(Set dst src);
6885
6886 ins_cost(INSN_COST);
6887 format %{ "mov $dst, $src\t# int" %}
6888
6889 ins_encode( aarch64_enc_movw_imm(dst, src) );
6890
6891 ins_pipe(ialu_imm);
6892 %}
6893
6894 // Load Long Constant
6895 instruct loadConL(iRegLNoSp dst, immL src)
6896 %{
6897 match(Set dst src);
6898
6899 ins_cost(INSN_COST);
6900 format %{ "mov $dst, $src\t# long" %}
6901
6902 ins_encode( aarch64_enc_mov_imm(dst, src) );
6903
6904 ins_pipe(ialu_imm);
6905 %}
6906
6907 // Load Pointer Constant
6908
6909 instruct loadConP(iRegPNoSp dst, immP con)
6910 %{
6911 match(Set dst con);
6912
6913 ins_cost(INSN_COST * 4);
6914 format %{
6915 "mov $dst, $con\t# ptr\n\t"
6916 %}
6917
6918 ins_encode(aarch64_enc_mov_p(dst, con));
6919
6920 ins_pipe(ialu_imm);
6921 %}
6922
6923 // Load Null Pointer Constant
6924
6925 instruct loadConP0(iRegPNoSp dst, immP0 con)
6926 %{
6927 match(Set dst con);
6928
6929 ins_cost(INSN_COST);
6930 format %{ "mov $dst, $con\t# nullptr ptr" %}
6931
6932 ins_encode(aarch64_enc_mov_p0(dst, con));
6933
6934 ins_pipe(ialu_imm);
6935 %}
6936
6937 // Load Pointer Constant One
6938
6939 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6940 %{
6941 match(Set dst con);
6942
6943 ins_cost(INSN_COST);
6944 format %{ "mov $dst, $con\t# nullptr ptr" %}
6945
6946 ins_encode(aarch64_enc_mov_p1(dst, con));
6947
6948 ins_pipe(ialu_imm);
6949 %}
6950
6951 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6952 %{
6953 match(Set dst con);
6954
6955 ins_cost(INSN_COST);
6956 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6957
6958 ins_encode %{
6959 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6960 %}
6961
6962 ins_pipe(ialu_imm);
6963 %}
6964
6965 // Load Narrow Pointer Constant
6966
6967 instruct loadConN(iRegNNoSp dst, immN con)
6968 %{
6969 match(Set dst con);
6970
6971 ins_cost(INSN_COST * 4);
6972 format %{ "mov $dst, $con\t# compressed ptr" %}
6973
6974 ins_encode(aarch64_enc_mov_n(dst, con));
6975
6976 ins_pipe(ialu_imm);
6977 %}
6978
6979 // Load Narrow Null Pointer Constant
6980
6981 instruct loadConN0(iRegNNoSp dst, immN0 con)
6982 %{
6983 match(Set dst con);
6984
6985 ins_cost(INSN_COST);
6986 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6987
6988 ins_encode(aarch64_enc_mov_n0(dst, con));
6989
6990 ins_pipe(ialu_imm);
6991 %}
6992
6993 // Load Narrow Klass Constant
6994
6995 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6996 %{
6997 match(Set dst con);
6998
6999 ins_cost(INSN_COST);
7000 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7001
7002 ins_encode(aarch64_enc_mov_nk(dst, con));
7003
7004 ins_pipe(ialu_imm);
7005 %}
7006
7007 // Load Packed Float Constant
7008
7009 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7010 match(Set dst con);
7011 ins_cost(INSN_COST * 4);
7012 format %{ "fmovs $dst, $con"%}
7013 ins_encode %{
7014 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7015 %}
7016
7017 ins_pipe(fp_imm_s);
7018 %}
7019
7020 // Load Float Constant
7021
7022 instruct loadConF(vRegF dst, immF con) %{
7023 match(Set dst con);
7024
7025 ins_cost(INSN_COST * 4);
7026
7027 format %{
7028 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7029 %}
7030
7031 ins_encode %{
7032 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7033 %}
7034
7035 ins_pipe(fp_load_constant_s);
7036 %}
7037
7038 // Load Packed Double Constant
7039
7040 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7041 match(Set dst con);
7042 ins_cost(INSN_COST);
7043 format %{ "fmovd $dst, $con"%}
7044 ins_encode %{
7045 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7046 %}
7047
7048 ins_pipe(fp_imm_d);
7049 %}
7050
7051 // Load Double Constant
7052
7053 instruct loadConD(vRegD dst, immD con) %{
7054 match(Set dst con);
7055
7056 ins_cost(INSN_COST * 5);
7057 format %{
7058 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7059 %}
7060
7061 ins_encode %{
7062 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7063 %}
7064
7065 ins_pipe(fp_load_constant_d);
7066 %}
7067
7068 // Load Half Float Constant
7069 instruct loadConH(vRegF dst, immH con) %{
7070 match(Set dst con);
7071 format %{ "mov rscratch1, $con\n\t"
7072 "fmov $dst, rscratch1"
7073 %}
7074 ins_encode %{
7075 __ movw(rscratch1, (uint32_t)$con$$constant);
7076 __ fmovs($dst$$FloatRegister, rscratch1);
7077 %}
7078 ins_pipe(pipe_class_default);
7079 %}
7080
7081 // Store Instructions
7082
7083 // Store Byte
7084 instruct storeB(iRegIorL2I src, memory1 mem)
7085 %{
7086 match(Set mem (StoreB mem src));
7087 predicate(!needs_releasing_store(n));
7088
7089 ins_cost(INSN_COST);
7090 format %{ "strb $src, $mem\t# byte" %}
7091
7092 ins_encode(aarch64_enc_strb(src, mem));
7093
7094 ins_pipe(istore_reg_mem);
7095 %}
7096
7097
7098 instruct storeimmB0(immI0 zero, memory1 mem)
7099 %{
7100 match(Set mem (StoreB mem zero));
7101 predicate(!needs_releasing_store(n));
7102
7103 ins_cost(INSN_COST);
7104 format %{ "strb rscractch2, $mem\t# byte" %}
7105
7106 ins_encode(aarch64_enc_strb0(mem));
7107
7108 ins_pipe(istore_mem);
7109 %}
7110
7111 // Store Char/Short
7112 instruct storeC(iRegIorL2I src, memory2 mem)
7113 %{
7114 match(Set mem (StoreC mem src));
7115 predicate(!needs_releasing_store(n));
7116
7117 ins_cost(INSN_COST);
7118 format %{ "strh $src, $mem\t# short" %}
7119
7120 ins_encode(aarch64_enc_strh(src, mem));
7121
7122 ins_pipe(istore_reg_mem);
7123 %}
7124
7125 instruct storeimmC0(immI0 zero, memory2 mem)
7126 %{
7127 match(Set mem (StoreC mem zero));
7128 predicate(!needs_releasing_store(n));
7129
7130 ins_cost(INSN_COST);
7131 format %{ "strh zr, $mem\t# short" %}
7132
7133 ins_encode(aarch64_enc_strh0(mem));
7134
7135 ins_pipe(istore_mem);
7136 %}
7137
7138 // Store Integer
7139
7140 instruct storeI(iRegIorL2I src, memory4 mem)
7141 %{
7142 match(Set mem(StoreI mem src));
7143 predicate(!needs_releasing_store(n));
7144
7145 ins_cost(INSN_COST);
7146 format %{ "strw $src, $mem\t# int" %}
7147
7148 ins_encode(aarch64_enc_strw(src, mem));
7149
7150 ins_pipe(istore_reg_mem);
7151 %}
7152
7153 instruct storeimmI0(immI0 zero, memory4 mem)
7154 %{
7155 match(Set mem(StoreI mem zero));
7156 predicate(!needs_releasing_store(n));
7157
7158 ins_cost(INSN_COST);
7159 format %{ "strw zr, $mem\t# int" %}
7160
7161 ins_encode(aarch64_enc_strw0(mem));
7162
7163 ins_pipe(istore_mem);
7164 %}
7165
7166 // Store Long (64 bit signed)
7167 instruct storeL(iRegL src, memory8 mem)
7168 %{
7169 match(Set mem (StoreL mem src));
7170 predicate(!needs_releasing_store(n));
7171
7172 ins_cost(INSN_COST);
7173 format %{ "str $src, $mem\t# int" %}
7174
7175 ins_encode(aarch64_enc_str(src, mem));
7176
7177 ins_pipe(istore_reg_mem);
7178 %}
7179
7180 // Store Long (64 bit signed)
7181 instruct storeimmL0(immL0 zero, memory8 mem)
7182 %{
7183 match(Set mem (StoreL mem zero));
7184 predicate(!needs_releasing_store(n));
7185
7186 ins_cost(INSN_COST);
7187 format %{ "str zr, $mem\t# int" %}
7188
7189 ins_encode(aarch64_enc_str0(mem));
7190
7191 ins_pipe(istore_mem);
7192 %}
7193
7194 // Store Pointer
7195 instruct storeP(iRegP src, memory8 mem)
7196 %{
7197 match(Set mem (StoreP mem src));
7198 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7199
7200 ins_cost(INSN_COST);
7201 format %{ "str $src, $mem\t# ptr" %}
7202
7203 ins_encode(aarch64_enc_str(src, mem));
7204
7205 ins_pipe(istore_reg_mem);
7206 %}
7207
7208 // Store Pointer
7209 instruct storeimmP0(immP0 zero, memory8 mem)
7210 %{
7211 match(Set mem (StoreP mem zero));
7212 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7213
7214 ins_cost(INSN_COST);
7215 format %{ "str zr, $mem\t# ptr" %}
7216
7217 ins_encode(aarch64_enc_str0(mem));
7218
7219 ins_pipe(istore_mem);
7220 %}
7221
7222 // Store Compressed Pointer
7223 instruct storeN(iRegN src, memory4 mem)
7224 %{
7225 match(Set mem (StoreN mem src));
7226 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7227
7228 ins_cost(INSN_COST);
7229 format %{ "strw $src, $mem\t# compressed ptr" %}
7230
7231 ins_encode(aarch64_enc_strw(src, mem));
7232
7233 ins_pipe(istore_reg_mem);
7234 %}
7235
7236 instruct storeImmN0(immN0 zero, memory4 mem)
7237 %{
7238 match(Set mem (StoreN mem zero));
7239 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7240
7241 ins_cost(INSN_COST);
7242 format %{ "strw zr, $mem\t# compressed ptr" %}
7243
7244 ins_encode(aarch64_enc_strw0(mem));
7245
7246 ins_pipe(istore_mem);
7247 %}
7248
7249 // Store Float
7250 instruct storeF(vRegF src, memory4 mem)
7251 %{
7252 match(Set mem (StoreF mem src));
7253 predicate(!needs_releasing_store(n));
7254
7255 ins_cost(INSN_COST);
7256 format %{ "strs $src, $mem\t# float" %}
7257
7258 ins_encode( aarch64_enc_strs(src, mem) );
7259
7260 ins_pipe(pipe_class_memory);
7261 %}
7262
7263 // TODO
7264 // implement storeImmF0 and storeFImmPacked
7265
7266 // Store Double
7267 instruct storeD(vRegD src, memory8 mem)
7268 %{
7269 match(Set mem (StoreD mem src));
7270 predicate(!needs_releasing_store(n));
7271
7272 ins_cost(INSN_COST);
7273 format %{ "strd $src, $mem\t# double" %}
7274
7275 ins_encode( aarch64_enc_strd(src, mem) );
7276
7277 ins_pipe(pipe_class_memory);
7278 %}
7279
7280 // Store Compressed Klass Pointer
7281 instruct storeNKlass(iRegN src, memory4 mem)
7282 %{
7283 predicate(!needs_releasing_store(n));
7284 match(Set mem (StoreNKlass mem src));
7285
7286 ins_cost(INSN_COST);
7287 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7288
7289 ins_encode(aarch64_enc_strw(src, mem));
7290
7291 ins_pipe(istore_reg_mem);
7292 %}
7293
7294 // TODO
7295 // implement storeImmD0 and storeDImmPacked
7296
7297 // prefetch instructions
7298 // Must be safe to execute with invalid address (cannot fault).
7299
7300 instruct prefetchalloc( memory8 mem ) %{
7301 match(PrefetchAllocation mem);
7302
7303 ins_cost(INSN_COST);
7304 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7305
7306 ins_encode( aarch64_enc_prefetchw(mem) );
7307
7308 ins_pipe(iload_prefetch);
7309 %}
7310
7311 // ---------------- volatile loads and stores ----------------
7312
7313 // Load Byte (8 bit signed)
7314 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7315 %{
7316 match(Set dst (LoadB mem));
7317
7318 ins_cost(VOLATILE_REF_COST);
7319 format %{ "ldarsb $dst, $mem\t# byte" %}
7320
7321 ins_encode(aarch64_enc_ldarsb(dst, mem));
7322
7323 ins_pipe(pipe_serial);
7324 %}
7325
7326 // Load Byte (8 bit signed) into long
7327 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7328 %{
7329 match(Set dst (ConvI2L (LoadB mem)));
7330
7331 ins_cost(VOLATILE_REF_COST);
7332 format %{ "ldarsb $dst, $mem\t# byte" %}
7333
7334 ins_encode(aarch64_enc_ldarsb(dst, mem));
7335
7336 ins_pipe(pipe_serial);
7337 %}
7338
7339 // Load Byte (8 bit unsigned)
7340 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7341 %{
7342 match(Set dst (LoadUB mem));
7343
7344 ins_cost(VOLATILE_REF_COST);
7345 format %{ "ldarb $dst, $mem\t# byte" %}
7346
7347 ins_encode(aarch64_enc_ldarb(dst, mem));
7348
7349 ins_pipe(pipe_serial);
7350 %}
7351
7352 // Load Byte (8 bit unsigned) into long
7353 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7354 %{
7355 match(Set dst (ConvI2L (LoadUB mem)));
7356
7357 ins_cost(VOLATILE_REF_COST);
7358 format %{ "ldarb $dst, $mem\t# byte" %}
7359
7360 ins_encode(aarch64_enc_ldarb(dst, mem));
7361
7362 ins_pipe(pipe_serial);
7363 %}
7364
7365 // Load Short (16 bit signed)
7366 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7367 %{
7368 match(Set dst (LoadS mem));
7369
7370 ins_cost(VOLATILE_REF_COST);
7371 format %{ "ldarshw $dst, $mem\t# short" %}
7372
7373 ins_encode(aarch64_enc_ldarshw(dst, mem));
7374
7375 ins_pipe(pipe_serial);
7376 %}
7377
7378 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7379 %{
7380 match(Set dst (LoadUS mem));
7381
7382 ins_cost(VOLATILE_REF_COST);
7383 format %{ "ldarhw $dst, $mem\t# short" %}
7384
7385 ins_encode(aarch64_enc_ldarhw(dst, mem));
7386
7387 ins_pipe(pipe_serial);
7388 %}
7389
7390 // Load Short/Char (16 bit unsigned) into long
7391 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7392 %{
7393 match(Set dst (ConvI2L (LoadUS mem)));
7394
7395 ins_cost(VOLATILE_REF_COST);
7396 format %{ "ldarh $dst, $mem\t# short" %}
7397
7398 ins_encode(aarch64_enc_ldarh(dst, mem));
7399
7400 ins_pipe(pipe_serial);
7401 %}
7402
7403 // Load Short/Char (16 bit signed) into long
7404 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7405 %{
7406 match(Set dst (ConvI2L (LoadS mem)));
7407
7408 ins_cost(VOLATILE_REF_COST);
7409 format %{ "ldarh $dst, $mem\t# short" %}
7410
7411 ins_encode(aarch64_enc_ldarsh(dst, mem));
7412
7413 ins_pipe(pipe_serial);
7414 %}
7415
7416 // Load Integer (32 bit signed)
7417 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7418 %{
7419 match(Set dst (LoadI mem));
7420
7421 ins_cost(VOLATILE_REF_COST);
7422 format %{ "ldarw $dst, $mem\t# int" %}
7423
7424 ins_encode(aarch64_enc_ldarw(dst, mem));
7425
7426 ins_pipe(pipe_serial);
7427 %}
7428
7429 // Load Integer (32 bit unsigned) into long
7430 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7431 %{
7432 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7433
7434 ins_cost(VOLATILE_REF_COST);
7435 format %{ "ldarw $dst, $mem\t# int" %}
7436
7437 ins_encode(aarch64_enc_ldarw(dst, mem));
7438
7439 ins_pipe(pipe_serial);
7440 %}
7441
7442 // Load Long (64 bit signed)
7443 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7444 %{
7445 match(Set dst (LoadL mem));
7446
7447 ins_cost(VOLATILE_REF_COST);
7448 format %{ "ldar $dst, $mem\t# int" %}
7449
7450 ins_encode(aarch64_enc_ldar(dst, mem));
7451
7452 ins_pipe(pipe_serial);
7453 %}
7454
7455 // Load Pointer
7456 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7457 %{
7458 match(Set dst (LoadP mem));
7459 predicate(n->as_Load()->barrier_data() == 0);
7460
7461 ins_cost(VOLATILE_REF_COST);
7462 format %{ "ldar $dst, $mem\t# ptr" %}
7463
7464 ins_encode(aarch64_enc_ldar(dst, mem));
7465
7466 ins_pipe(pipe_serial);
7467 %}
7468
7469 // Load Compressed Pointer
7470 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7471 %{
7472 match(Set dst (LoadN mem));
7473 predicate(n->as_Load()->barrier_data() == 0);
7474
7475 ins_cost(VOLATILE_REF_COST);
7476 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7477
7478 ins_encode(aarch64_enc_ldarw(dst, mem));
7479
7480 ins_pipe(pipe_serial);
7481 %}
7482
7483 // Load Float
7484 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7485 %{
7486 match(Set dst (LoadF mem));
7487
7488 ins_cost(VOLATILE_REF_COST);
7489 format %{ "ldars $dst, $mem\t# float" %}
7490
7491 ins_encode( aarch64_enc_fldars(dst, mem) );
7492
7493 ins_pipe(pipe_serial);
7494 %}
7495
7496 // Load Double
7497 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7498 %{
7499 match(Set dst (LoadD mem));
7500
7501 ins_cost(VOLATILE_REF_COST);
7502 format %{ "ldard $dst, $mem\t# double" %}
7503
7504 ins_encode( aarch64_enc_fldard(dst, mem) );
7505
7506 ins_pipe(pipe_serial);
7507 %}
7508
7509 // Store Byte
7510 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7511 %{
7512 match(Set mem (StoreB mem src));
7513
7514 ins_cost(VOLATILE_REF_COST);
7515 format %{ "stlrb $src, $mem\t# byte" %}
7516
7517 ins_encode(aarch64_enc_stlrb(src, mem));
7518
7519 ins_pipe(pipe_class_memory);
7520 %}
7521
7522 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7523 %{
7524 match(Set mem (StoreB mem zero));
7525
7526 ins_cost(VOLATILE_REF_COST);
7527 format %{ "stlrb zr, $mem\t# byte" %}
7528
7529 ins_encode(aarch64_enc_stlrb0(mem));
7530
7531 ins_pipe(pipe_class_memory);
7532 %}
7533
7534 // Store Char/Short
7535 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7536 %{
7537 match(Set mem (StoreC mem src));
7538
7539 ins_cost(VOLATILE_REF_COST);
7540 format %{ "stlrh $src, $mem\t# short" %}
7541
7542 ins_encode(aarch64_enc_stlrh(src, mem));
7543
7544 ins_pipe(pipe_class_memory);
7545 %}
7546
7547 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7548 %{
7549 match(Set mem (StoreC mem zero));
7550
7551 ins_cost(VOLATILE_REF_COST);
7552 format %{ "stlrh zr, $mem\t# short" %}
7553
7554 ins_encode(aarch64_enc_stlrh0(mem));
7555
7556 ins_pipe(pipe_class_memory);
7557 %}
7558
7559 // Store Integer
7560
7561 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7562 %{
7563 match(Set mem(StoreI mem src));
7564
7565 ins_cost(VOLATILE_REF_COST);
7566 format %{ "stlrw $src, $mem\t# int" %}
7567
7568 ins_encode(aarch64_enc_stlrw(src, mem));
7569
7570 ins_pipe(pipe_class_memory);
7571 %}
7572
7573 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7574 %{
7575 match(Set mem(StoreI mem zero));
7576
7577 ins_cost(VOLATILE_REF_COST);
7578 format %{ "stlrw zr, $mem\t# int" %}
7579
7580 ins_encode(aarch64_enc_stlrw0(mem));
7581
7582 ins_pipe(pipe_class_memory);
7583 %}
7584
7585 // Store Long (64 bit signed)
7586 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7587 %{
7588 match(Set mem (StoreL mem src));
7589
7590 ins_cost(VOLATILE_REF_COST);
7591 format %{ "stlr $src, $mem\t# int" %}
7592
7593 ins_encode(aarch64_enc_stlr(src, mem));
7594
7595 ins_pipe(pipe_class_memory);
7596 %}
7597
7598 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7599 %{
7600 match(Set mem (StoreL mem zero));
7601
7602 ins_cost(VOLATILE_REF_COST);
7603 format %{ "stlr zr, $mem\t# int" %}
7604
7605 ins_encode(aarch64_enc_stlr0(mem));
7606
7607 ins_pipe(pipe_class_memory);
7608 %}
7609
7610 // Store Pointer
7611 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7612 %{
7613 match(Set mem (StoreP mem src));
7614 predicate(n->as_Store()->barrier_data() == 0);
7615
7616 ins_cost(VOLATILE_REF_COST);
7617 format %{ "stlr $src, $mem\t# ptr" %}
7618
7619 ins_encode(aarch64_enc_stlr(src, mem));
7620
7621 ins_pipe(pipe_class_memory);
7622 %}
7623
7624 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7625 %{
7626 match(Set mem (StoreP mem zero));
7627 predicate(n->as_Store()->barrier_data() == 0);
7628
7629 ins_cost(VOLATILE_REF_COST);
7630 format %{ "stlr zr, $mem\t# ptr" %}
7631
7632 ins_encode(aarch64_enc_stlr0(mem));
7633
7634 ins_pipe(pipe_class_memory);
7635 %}
7636
7637 // Store Compressed Pointer
7638 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7639 %{
7640 match(Set mem (StoreN mem src));
7641 predicate(n->as_Store()->barrier_data() == 0);
7642
7643 ins_cost(VOLATILE_REF_COST);
7644 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7645
7646 ins_encode(aarch64_enc_stlrw(src, mem));
7647
7648 ins_pipe(pipe_class_memory);
7649 %}
7650
7651 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7652 %{
7653 match(Set mem (StoreN mem zero));
7654 predicate(n->as_Store()->barrier_data() == 0);
7655
7656 ins_cost(VOLATILE_REF_COST);
7657 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7658
7659 ins_encode(aarch64_enc_stlrw0(mem));
7660
7661 ins_pipe(pipe_class_memory);
7662 %}
7663
7664 // Store Float
7665 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7666 %{
7667 match(Set mem (StoreF mem src));
7668
7669 ins_cost(VOLATILE_REF_COST);
7670 format %{ "stlrs $src, $mem\t# float" %}
7671
7672 ins_encode( aarch64_enc_fstlrs(src, mem) );
7673
7674 ins_pipe(pipe_class_memory);
7675 %}
7676
7677 // TODO
7678 // implement storeImmF0 and storeFImmPacked
7679
7680 // Store Double
7681 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7682 %{
7683 match(Set mem (StoreD mem src));
7684
7685 ins_cost(VOLATILE_REF_COST);
7686 format %{ "stlrd $src, $mem\t# double" %}
7687
7688 ins_encode( aarch64_enc_fstlrd(src, mem) );
7689
7690 ins_pipe(pipe_class_memory);
7691 %}
7692
7693 // ---------------- end of volatile loads and stores ----------------
7694
7695 instruct cacheWB(indirect addr)
7696 %{
7697 predicate(VM_Version::supports_data_cache_line_flush());
7698 match(CacheWB addr);
7699
7700 ins_cost(100);
7701 format %{"cache wb $addr" %}
7702 ins_encode %{
7703 assert($addr->index_position() < 0, "should be");
7704 assert($addr$$disp == 0, "should be");
7705 __ cache_wb(Address($addr$$base$$Register, 0));
7706 %}
7707 ins_pipe(pipe_slow); // XXX
7708 %}
7709
7710 instruct cacheWBPreSync()
7711 %{
7712 predicate(VM_Version::supports_data_cache_line_flush());
7713 match(CacheWBPreSync);
7714
7715 ins_cost(100);
7716 format %{"cache wb presync" %}
7717 ins_encode %{
7718 __ cache_wbsync(true);
7719 %}
7720 ins_pipe(pipe_slow); // XXX
7721 %}
7722
7723 instruct cacheWBPostSync()
7724 %{
7725 predicate(VM_Version::supports_data_cache_line_flush());
7726 match(CacheWBPostSync);
7727
7728 ins_cost(100);
7729 format %{"cache wb postsync" %}
7730 ins_encode %{
7731 __ cache_wbsync(false);
7732 %}
7733 ins_pipe(pipe_slow); // XXX
7734 %}
7735
7736 // ============================================================================
7737 // BSWAP Instructions
7738
7739 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7740 match(Set dst (ReverseBytesI src));
7741
7742 ins_cost(INSN_COST);
7743 format %{ "revw $dst, $src" %}
7744
7745 ins_encode %{
7746 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7747 %}
7748
7749 ins_pipe(ialu_reg);
7750 %}
7751
7752 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7753 match(Set dst (ReverseBytesL src));
7754
7755 ins_cost(INSN_COST);
7756 format %{ "rev $dst, $src" %}
7757
7758 ins_encode %{
7759 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7760 %}
7761
7762 ins_pipe(ialu_reg);
7763 %}
7764
7765 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7766 match(Set dst (ReverseBytesUS src));
7767
7768 ins_cost(INSN_COST);
7769 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7770
7771 ins_encode %{
7772 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7773 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7774 %}
7775
7776 ins_pipe(ialu_reg);
7777 %}
7778
7779 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7780 match(Set dst (ReverseBytesS src));
7781
7782 ins_cost(INSN_COST);
7783 format %{ "rev16w $dst, $src\n\t"
7784 "sbfmw $dst, $dst, #0, #15" %}
7785
7786 ins_encode %{
7787 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7788 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7789 %}
7790
7791 ins_pipe(ialu_reg);
7792 %}
7793
7794 // ============================================================================
7795 // Zero Count Instructions
7796
7797 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7798 match(Set dst (CountLeadingZerosI src));
7799
7800 ins_cost(INSN_COST);
7801 format %{ "clzw $dst, $src" %}
7802 ins_encode %{
7803 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7804 %}
7805
7806 ins_pipe(ialu_reg);
7807 %}
7808
7809 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7810 match(Set dst (CountLeadingZerosL src));
7811
7812 ins_cost(INSN_COST);
7813 format %{ "clz $dst, $src" %}
7814 ins_encode %{
7815 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7816 %}
7817
7818 ins_pipe(ialu_reg);
7819 %}
7820
7821 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7822 match(Set dst (CountTrailingZerosI src));
7823
7824 ins_cost(INSN_COST * 2);
7825 format %{ "rbitw $dst, $src\n\t"
7826 "clzw $dst, $dst" %}
7827 ins_encode %{
7828 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7829 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7830 %}
7831
7832 ins_pipe(ialu_reg);
7833 %}
7834
7835 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7836 match(Set dst (CountTrailingZerosL src));
7837
7838 ins_cost(INSN_COST * 2);
7839 format %{ "rbit $dst, $src\n\t"
7840 "clz $dst, $dst" %}
7841 ins_encode %{
7842 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7843 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7844 %}
7845
7846 ins_pipe(ialu_reg);
7847 %}
7848
7849 //---------- Population Count Instructions -------------------------------------
7850 //
7851
7852 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7853 match(Set dst (PopCountI src));
7854 effect(TEMP tmp);
7855 ins_cost(INSN_COST * 13);
7856
7857 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7858 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7859 "addv $tmp, $tmp\t# vector (8B)\n\t"
7860 "mov $dst, $tmp\t# vector (1D)" %}
7861 ins_encode %{
7862 __ fmovs($tmp$$FloatRegister, $src$$Register);
7863 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7864 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7865 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7866 %}
7867
7868 ins_pipe(pipe_class_default);
7869 %}
7870
7871 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7872 match(Set dst (PopCountI (LoadI mem)));
7873 effect(TEMP tmp);
7874 ins_cost(INSN_COST * 13);
7875
7876 format %{ "ldrs $tmp, $mem\n\t"
7877 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7878 "addv $tmp, $tmp\t# vector (8B)\n\t"
7879 "mov $dst, $tmp\t# vector (1D)" %}
7880 ins_encode %{
7881 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7882 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7883 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7884 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7885 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7886 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7887 %}
7888
7889 ins_pipe(pipe_class_default);
7890 %}
7891
7892 // Note: Long.bitCount(long) returns an int.
7893 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7894 match(Set dst (PopCountL src));
7895 effect(TEMP tmp);
7896 ins_cost(INSN_COST * 13);
7897
7898 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7899 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7900 "addv $tmp, $tmp\t# vector (8B)\n\t"
7901 "mov $dst, $tmp\t# vector (1D)" %}
7902 ins_encode %{
7903 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7904 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7905 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7906 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7907 %}
7908
7909 ins_pipe(pipe_class_default);
7910 %}
7911
7912 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7913 match(Set dst (PopCountL (LoadL mem)));
7914 effect(TEMP tmp);
7915 ins_cost(INSN_COST * 13);
7916
7917 format %{ "ldrd $tmp, $mem\n\t"
7918 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7919 "addv $tmp, $tmp\t# vector (8B)\n\t"
7920 "mov $dst, $tmp\t# vector (1D)" %}
7921 ins_encode %{
7922 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7923 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7924 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7925 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7926 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7927 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7928 %}
7929
7930 ins_pipe(pipe_class_default);
7931 %}
7932
7933 // ============================================================================
7934 // VerifyVectorAlignment Instruction
7935
7936 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7937 match(Set addr (VerifyVectorAlignment addr mask));
7938 effect(KILL cr);
7939 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7940 ins_encode %{
7941 Label Lskip;
7942 // check if masked bits of addr are zero
7943 __ tst($addr$$Register, $mask$$constant);
7944 __ br(Assembler::EQ, Lskip);
7945 __ stop("verify_vector_alignment found a misaligned vector memory access");
7946 __ bind(Lskip);
7947 %}
7948 ins_pipe(pipe_slow);
7949 %}
7950
7951 // ============================================================================
7952 // MemBar Instruction
7953
7954 instruct load_fence() %{
7955 match(LoadFence);
7956 ins_cost(VOLATILE_REF_COST);
7957
7958 format %{ "load_fence" %}
7959
7960 ins_encode %{
7961 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7962 %}
7963 ins_pipe(pipe_serial);
7964 %}
7965
7966 instruct unnecessary_membar_acquire() %{
7967 predicate(unnecessary_acquire(n));
7968 match(MemBarAcquire);
7969 ins_cost(0);
7970
7971 format %{ "membar_acquire (elided)" %}
7972
7973 ins_encode %{
7974 __ block_comment("membar_acquire (elided)");
7975 %}
7976
7977 ins_pipe(pipe_class_empty);
7978 %}
7979
7980 instruct membar_acquire() %{
7981 match(MemBarAcquire);
7982 ins_cost(VOLATILE_REF_COST);
7983
7984 format %{ "membar_acquire\n\t"
7985 "dmb ishld" %}
7986
7987 ins_encode %{
7988 __ block_comment("membar_acquire");
7989 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7990 %}
7991
7992 ins_pipe(pipe_serial);
7993 %}
7994
7995
7996 instruct membar_acquire_lock() %{
7997 match(MemBarAcquireLock);
7998 ins_cost(VOLATILE_REF_COST);
7999
8000 format %{ "membar_acquire_lock (elided)" %}
8001
8002 ins_encode %{
8003 __ block_comment("membar_acquire_lock (elided)");
8004 %}
8005
8006 ins_pipe(pipe_serial);
8007 %}
8008
8009 instruct store_fence() %{
8010 match(StoreFence);
8011 ins_cost(VOLATILE_REF_COST);
8012
8013 format %{ "store_fence" %}
8014
8015 ins_encode %{
8016 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8017 %}
8018 ins_pipe(pipe_serial);
8019 %}
8020
8021 instruct unnecessary_membar_release() %{
8022 predicate(unnecessary_release(n));
8023 match(MemBarRelease);
8024 ins_cost(0);
8025
8026 format %{ "membar_release (elided)" %}
8027
8028 ins_encode %{
8029 __ block_comment("membar_release (elided)");
8030 %}
8031 ins_pipe(pipe_serial);
8032 %}
8033
8034 instruct membar_release() %{
8035 match(MemBarRelease);
8036 ins_cost(VOLATILE_REF_COST);
8037
8038 format %{ "membar_release\n\t"
8039 "dmb ishst\n\tdmb ishld" %}
8040
8041 ins_encode %{
8042 __ block_comment("membar_release");
8043 // These will be merged if AlwaysMergeDMB is enabled.
8044 __ membar(Assembler::StoreStore);
8045 __ membar(Assembler::LoadStore);
8046 %}
8047 ins_pipe(pipe_serial);
8048 %}
8049
8050 instruct membar_storestore() %{
8051 match(MemBarStoreStore);
8052 match(StoreStoreFence);
8053 ins_cost(VOLATILE_REF_COST);
8054
8055 format %{ "MEMBAR-store-store" %}
8056
8057 ins_encode %{
8058 __ membar(Assembler::StoreStore);
8059 %}
8060 ins_pipe(pipe_serial);
8061 %}
8062
8063 instruct membar_release_lock() %{
8064 match(MemBarReleaseLock);
8065 ins_cost(VOLATILE_REF_COST);
8066
8067 format %{ "membar_release_lock (elided)" %}
8068
8069 ins_encode %{
8070 __ block_comment("membar_release_lock (elided)");
8071 %}
8072
8073 ins_pipe(pipe_serial);
8074 %}
8075
8076 instruct membar_storeload() %{
8077 match(MemBarStoreLoad);
8078 ins_cost(VOLATILE_REF_COST*100);
8079
8080 format %{ "MEMBAR-store-load\n\t"
8081 "dmb ish" %}
8082
8083 ins_encode %{
8084 __ block_comment("membar_storeload");
8085 __ membar(Assembler::StoreLoad);
8086 %}
8087
8088 ins_pipe(pipe_serial);
8089 %}
8090
8091 instruct unnecessary_membar_volatile() %{
8092 predicate(unnecessary_volatile(n));
8093 match(MemBarVolatile);
8094 ins_cost(0);
8095
8096 format %{ "membar_volatile (elided)" %}
8097
8098 ins_encode %{
8099 __ block_comment("membar_volatile (elided)");
8100 %}
8101
8102 ins_pipe(pipe_serial);
8103 %}
8104
8105 instruct membar_volatile() %{
8106 match(MemBarVolatile);
8107 ins_cost(VOLATILE_REF_COST*100);
8108
8109 format %{ "membar_volatile\n\t"
8110 "dmb ish"%}
8111
8112 ins_encode %{
8113 __ block_comment("membar_volatile");
8114 __ membar(Assembler::StoreLoad);
8115 %}
8116
8117 ins_pipe(pipe_serial);
8118 %}
8119
8120 instruct membar_full() %{
8121 match(MemBarFull);
8122 ins_cost(VOLATILE_REF_COST*100);
8123
8124 format %{ "membar_full\n\t"
8125 "dmb ish" %}
8126 ins_encode %{
8127 __ block_comment("membar_full");
8128 __ membar(Assembler::AnyAny);
8129 %}
8130
8131 ins_pipe(pipe_serial);
8132 %}
8133
8134 // ============================================================================
8135 // Cast/Convert Instructions
8136
8137 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8138 match(Set dst (CastX2P src));
8139
8140 ins_cost(INSN_COST);
8141 format %{ "mov $dst, $src\t# long -> ptr" %}
8142
8143 ins_encode %{
8144 if ($dst$$reg != $src$$reg) {
8145 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8146 }
8147 %}
8148
8149 ins_pipe(ialu_reg);
8150 %}
8151
8152 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8153 match(Set dst (CastI2N src));
8154
8155 ins_cost(INSN_COST);
8156 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8157
8158 ins_encode %{
8159 if ($dst$$reg != $src$$reg) {
8160 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8161 }
8162 %}
8163
8164 ins_pipe(ialu_reg);
8165 %}
8166
8167 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8168 match(Set dst (CastP2X src));
8169
8170 ins_cost(INSN_COST);
8171 format %{ "mov $dst, $src\t# ptr -> long" %}
8172
8173 ins_encode %{
8174 if ($dst$$reg != $src$$reg) {
8175 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8176 }
8177 %}
8178
8179 ins_pipe(ialu_reg);
8180 %}
8181
8182 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8183 match(Set dst (CastP2X src));
8184
8185 ins_cost(INSN_COST);
8186 format %{ "mov $dst, $src\t# ptr -> long" %}
8187
8188 ins_encode %{
8189 if ($dst$$reg != $src$$reg) {
8190 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8191 }
8192 %}
8193
8194 ins_pipe(ialu_reg);
8195 %}
8196
8197 // Convert oop into int for vectors alignment masking
8198 instruct convP2I(iRegINoSp dst, iRegP src) %{
8199 match(Set dst (ConvL2I (CastP2X src)));
8200
8201 ins_cost(INSN_COST);
8202 format %{ "movw $dst, $src\t# ptr -> int" %}
8203 ins_encode %{
8204 __ movw($dst$$Register, $src$$Register);
8205 %}
8206
8207 ins_pipe(ialu_reg);
8208 %}
8209
8210 // Convert compressed oop into int for vectors alignment masking
8211 // in case of 32bit oops (heap < 4Gb).
8212 instruct convN2I(iRegINoSp dst, iRegN src)
8213 %{
8214 predicate(CompressedOops::shift() == 0);
8215 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8216
8217 ins_cost(INSN_COST);
8218 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8219 ins_encode %{
8220 __ movw($dst$$Register, $src$$Register);
8221 %}
8222
8223 ins_pipe(ialu_reg);
8224 %}
8225
8226
8227 // Convert oop pointer into compressed form
8228 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8229 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8230 match(Set dst (EncodeP src));
8231 effect(KILL cr);
8232 ins_cost(INSN_COST * 3);
8233 format %{ "encode_heap_oop $dst, $src" %}
8234 ins_encode %{
8235 Register s = $src$$Register;
8236 Register d = $dst$$Register;
8237 __ encode_heap_oop(d, s);
8238 %}
8239 ins_pipe(ialu_reg);
8240 %}
8241
8242 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8243 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8244 match(Set dst (EncodeP src));
8245 ins_cost(INSN_COST * 3);
8246 format %{ "encode_heap_oop_not_null $dst, $src" %}
8247 ins_encode %{
8248 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8249 %}
8250 ins_pipe(ialu_reg);
8251 %}
8252
8253 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8254 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8255 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8256 match(Set dst (DecodeN src));
8257 ins_cost(INSN_COST * 3);
8258 format %{ "decode_heap_oop $dst, $src" %}
8259 ins_encode %{
8260 Register s = $src$$Register;
8261 Register d = $dst$$Register;
8262 __ decode_heap_oop(d, s);
8263 %}
8264 ins_pipe(ialu_reg);
8265 %}
8266
8267 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8268 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8269 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8270 match(Set dst (DecodeN src));
8271 ins_cost(INSN_COST * 3);
8272 format %{ "decode_heap_oop_not_null $dst, $src" %}
8273 ins_encode %{
8274 Register s = $src$$Register;
8275 Register d = $dst$$Register;
8276 __ decode_heap_oop_not_null(d, s);
8277 %}
8278 ins_pipe(ialu_reg);
8279 %}
8280
8281 // n.b. AArch64 implementations of encode_klass_not_null and
8282 // decode_klass_not_null do not modify the flags register so, unlike
8283 // Intel, we don't kill CR as a side effect here
8284
8285 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8286 match(Set dst (EncodePKlass src));
8287
8288 ins_cost(INSN_COST * 3);
8289 format %{ "encode_klass_not_null $dst,$src" %}
8290
8291 ins_encode %{
8292 Register src_reg = as_Register($src$$reg);
8293 Register dst_reg = as_Register($dst$$reg);
8294 __ encode_klass_not_null(dst_reg, src_reg, rscratch1);
8295 %}
8296
8297 ins_pipe(ialu_reg);
8298 %}
8299
8300 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8301 match(Set dst (DecodeNKlass src));
8302
8303 ins_cost(INSN_COST * 3);
8304 format %{ "decode_klass_not_null $dst,$src" %}
8305
8306 ins_encode %{
8307 Register src_reg = as_Register($src$$reg);
8308 Register dst_reg = as_Register($dst$$reg);
8309 __ decode_klass_not_null(dst_reg, src_reg, rscratch1);
8310 %}
8311
8312 ins_pipe(ialu_reg);
8313 %}
8314
8315 instruct checkCastPP(iRegPNoSp dst)
8316 %{
8317 match(Set dst (CheckCastPP dst));
8318
8319 size(0);
8320 format %{ "# checkcastPP of $dst" %}
8321 ins_encode(/* empty encoding */);
8322 ins_pipe(pipe_class_empty);
8323 %}
8324
8325 instruct castPP(iRegPNoSp dst)
8326 %{
8327 match(Set dst (CastPP dst));
8328
8329 size(0);
8330 format %{ "# castPP of $dst" %}
8331 ins_encode(/* empty encoding */);
8332 ins_pipe(pipe_class_empty);
8333 %}
8334
8335 instruct castII(iRegI dst)
8336 %{
8337 predicate(VerifyConstraintCasts == 0);
8338 match(Set dst (CastII dst));
8339
8340 size(0);
8341 format %{ "# castII of $dst" %}
8342 ins_encode(/* empty encoding */);
8343 ins_cost(0);
8344 ins_pipe(pipe_class_empty);
8345 %}
8346
8347 instruct castII_checked(iRegI dst, rFlagsReg cr)
8348 %{
8349 predicate(VerifyConstraintCasts > 0);
8350 match(Set dst (CastII dst));
8351 effect(KILL cr);
8352
8353 format %{ "# castII_checked of $dst" %}
8354 ins_encode %{
8355 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8356 %}
8357 ins_pipe(pipe_slow);
8358 %}
8359
8360 // The unchecked and checked variants for CastII below both use iRegINoSp for src and dst
8361 // as some consumers of CastII node like ConvHF2F forbid the stack pointer as an input
8362 // (please see convHF2F_reg_reg rule which requires input to be in an iRegINoSp register).
8363 instruct castII_nosp(iRegINoSp dst)
8364 %{
8365 predicate(VerifyConstraintCasts == 0);
8366 match(Set dst (CastII dst));
8367
8368 size(0);
8369 format %{ "# castII of $dst" %}
8370 ins_encode(/* empty encoding */);
8371 ins_cost(0);
8372 ins_pipe(pipe_class_empty);
8373 %}
8374
8375 instruct castII_checked_nosp(iRegINoSp dst, rFlagsReg cr)
8376 %{
8377 predicate(VerifyConstraintCasts > 0);
8378 match(Set dst (CastII dst));
8379 effect(KILL cr);
8380
8381 format %{ "# castII_checked of $dst" %}
8382 ins_encode %{
8383 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8384 %}
8385 ins_pipe(pipe_slow);
8386 %}
8387
8388 instruct castLL(iRegL dst)
8389 %{
8390 predicate(VerifyConstraintCasts == 0);
8391 match(Set dst (CastLL dst));
8392
8393 size(0);
8394 format %{ "# castLL of $dst" %}
8395 ins_encode(/* empty encoding */);
8396 ins_cost(0);
8397 ins_pipe(pipe_class_empty);
8398 %}
8399
8400 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8401 %{
8402 predicate(VerifyConstraintCasts > 0);
8403 match(Set dst (CastLL dst));
8404 effect(KILL cr);
8405
8406 format %{ "# castLL_checked of $dst" %}
8407 ins_encode %{
8408 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8409 %}
8410 ins_pipe(pipe_slow);
8411 %}
8412
8413 instruct castHH(vRegF dst)
8414 %{
8415 match(Set dst (CastHH dst));
8416 size(0);
8417 format %{ "# castHH of $dst" %}
8418 ins_encode(/* empty encoding */);
8419 ins_cost(0);
8420 ins_pipe(pipe_class_empty);
8421 %}
8422
8423 instruct castFF(vRegF dst)
8424 %{
8425 match(Set dst (CastFF dst));
8426
8427 size(0);
8428 format %{ "# castFF of $dst" %}
8429 ins_encode(/* empty encoding */);
8430 ins_cost(0);
8431 ins_pipe(pipe_class_empty);
8432 %}
8433
8434 instruct castDD(vRegD dst)
8435 %{
8436 match(Set dst (CastDD dst));
8437
8438 size(0);
8439 format %{ "# castDD of $dst" %}
8440 ins_encode(/* empty encoding */);
8441 ins_cost(0);
8442 ins_pipe(pipe_class_empty);
8443 %}
8444
8445 instruct castVV(vReg dst)
8446 %{
8447 match(Set dst (CastVV dst));
8448
8449 size(0);
8450 format %{ "# castVV of $dst" %}
8451 ins_encode(/* empty encoding */);
8452 ins_cost(0);
8453 ins_pipe(pipe_class_empty);
8454 %}
8455
8456 instruct castVVMask(pRegGov dst)
8457 %{
8458 match(Set dst (CastVV dst));
8459
8460 size(0);
8461 format %{ "# castVV of $dst" %}
8462 ins_encode(/* empty encoding */);
8463 ins_cost(0);
8464 ins_pipe(pipe_class_empty);
8465 %}
8466
8467 // Manifest a CmpU result in an integer register.
8468 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8469 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8470 %{
8471 match(Set dst (CmpU3 src1 src2));
8472 effect(KILL flags);
8473
8474 ins_cost(INSN_COST * 3);
8475 format %{
8476 "cmpw $src1, $src2\n\t"
8477 "csetw $dst, ne\n\t"
8478 "cnegw $dst, lo\t# CmpU3(reg)"
8479 %}
8480 ins_encode %{
8481 __ cmpw($src1$$Register, $src2$$Register);
8482 __ csetw($dst$$Register, Assembler::NE);
8483 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8484 %}
8485
8486 ins_pipe(pipe_class_default);
8487 %}
8488
8489 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8490 %{
8491 match(Set dst (CmpU3 src1 src2));
8492 effect(KILL flags);
8493
8494 ins_cost(INSN_COST * 3);
8495 format %{
8496 "subsw zr, $src1, $src2\n\t"
8497 "csetw $dst, ne\n\t"
8498 "cnegw $dst, lo\t# CmpU3(imm)"
8499 %}
8500 ins_encode %{
8501 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8502 __ csetw($dst$$Register, Assembler::NE);
8503 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8504 %}
8505
8506 ins_pipe(pipe_class_default);
8507 %}
8508
8509 // Manifest a CmpUL result in an integer register.
8510 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8511 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8512 %{
8513 match(Set dst (CmpUL3 src1 src2));
8514 effect(KILL flags);
8515
8516 ins_cost(INSN_COST * 3);
8517 format %{
8518 "cmp $src1, $src2\n\t"
8519 "csetw $dst, ne\n\t"
8520 "cnegw $dst, lo\t# CmpUL3(reg)"
8521 %}
8522 ins_encode %{
8523 __ cmp($src1$$Register, $src2$$Register);
8524 __ csetw($dst$$Register, Assembler::NE);
8525 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8526 %}
8527
8528 ins_pipe(pipe_class_default);
8529 %}
8530
8531 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8532 %{
8533 match(Set dst (CmpUL3 src1 src2));
8534 effect(KILL flags);
8535
8536 ins_cost(INSN_COST * 3);
8537 format %{
8538 "subs zr, $src1, $src2\n\t"
8539 "csetw $dst, ne\n\t"
8540 "cnegw $dst, lo\t# CmpUL3(imm)"
8541 %}
8542 ins_encode %{
8543 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8544 __ csetw($dst$$Register, Assembler::NE);
8545 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8546 %}
8547
8548 ins_pipe(pipe_class_default);
8549 %}
8550
8551 // Manifest a CmpL result in an integer register.
8552 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8553 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8554 %{
8555 match(Set dst (CmpL3 src1 src2));
8556 effect(KILL flags);
8557
8558 ins_cost(INSN_COST * 3);
8559 format %{
8560 "cmp $src1, $src2\n\t"
8561 "csetw $dst, ne\n\t"
8562 "cnegw $dst, lt\t# CmpL3(reg)"
8563 %}
8564 ins_encode %{
8565 __ cmp($src1$$Register, $src2$$Register);
8566 __ csetw($dst$$Register, Assembler::NE);
8567 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8568 %}
8569
8570 ins_pipe(pipe_class_default);
8571 %}
8572
8573 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8574 %{
8575 match(Set dst (CmpL3 src1 src2));
8576 effect(KILL flags);
8577
8578 ins_cost(INSN_COST * 3);
8579 format %{
8580 "subs zr, $src1, $src2\n\t"
8581 "csetw $dst, ne\n\t"
8582 "cnegw $dst, lt\t# CmpL3(imm)"
8583 %}
8584 ins_encode %{
8585 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8586 __ csetw($dst$$Register, Assembler::NE);
8587 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8588 %}
8589
8590 ins_pipe(pipe_class_default);
8591 %}
8592
8593 // ============================================================================
8594 // Conditional Move Instructions
8595
8596 // n.b. we have identical rules for both a signed compare op (cmpOp)
8597 // and an unsigned compare op (cmpOpU). it would be nice if we could
8598 // define an op class which merged both inputs and use it to type the
8599 // argument to a single rule. unfortunatelyt his fails because the
8600 // opclass does not live up to the COND_INTER interface of its
8601 // component operands. When the generic code tries to negate the
8602 // operand it ends up running the generci Machoper::negate method
8603 // which throws a ShouldNotHappen. So, we have to provide two flavours
8604 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8605
8606 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8607 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8608
8609 ins_cost(INSN_COST * 2);
8610 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8611
8612 ins_encode %{
8613 __ cselw(as_Register($dst$$reg),
8614 as_Register($src2$$reg),
8615 as_Register($src1$$reg),
8616 (Assembler::Condition)$cmp$$cmpcode);
8617 %}
8618
8619 ins_pipe(icond_reg_reg);
8620 %}
8621
8622 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8623 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8624
8625 ins_cost(INSN_COST * 2);
8626 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8627
8628 ins_encode %{
8629 __ cselw(as_Register($dst$$reg),
8630 as_Register($src2$$reg),
8631 as_Register($src1$$reg),
8632 (Assembler::Condition)$cmp$$cmpcode);
8633 %}
8634
8635 ins_pipe(icond_reg_reg);
8636 %}
8637
8638 // special cases where one arg is zero
8639
8640 // n.b. this is selected in preference to the rule above because it
8641 // avoids loading constant 0 into a source register
8642
8643 // TODO
8644 // we ought only to be able to cull one of these variants as the ideal
8645 // transforms ought always to order the zero consistently (to left/right?)
8646
8647 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8648 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8649
8650 ins_cost(INSN_COST * 2);
8651 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8652
8653 ins_encode %{
8654 __ cselw(as_Register($dst$$reg),
8655 as_Register($src$$reg),
8656 zr,
8657 (Assembler::Condition)$cmp$$cmpcode);
8658 %}
8659
8660 ins_pipe(icond_reg);
8661 %}
8662
8663 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8664 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8665
8666 ins_cost(INSN_COST * 2);
8667 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8668
8669 ins_encode %{
8670 __ cselw(as_Register($dst$$reg),
8671 as_Register($src$$reg),
8672 zr,
8673 (Assembler::Condition)$cmp$$cmpcode);
8674 %}
8675
8676 ins_pipe(icond_reg);
8677 %}
8678
8679 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8680 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8681
8682 ins_cost(INSN_COST * 2);
8683 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8684
8685 ins_encode %{
8686 __ cselw(as_Register($dst$$reg),
8687 zr,
8688 as_Register($src$$reg),
8689 (Assembler::Condition)$cmp$$cmpcode);
8690 %}
8691
8692 ins_pipe(icond_reg);
8693 %}
8694
8695 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8696 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8697
8698 ins_cost(INSN_COST * 2);
8699 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8700
8701 ins_encode %{
8702 __ cselw(as_Register($dst$$reg),
8703 zr,
8704 as_Register($src$$reg),
8705 (Assembler::Condition)$cmp$$cmpcode);
8706 %}
8707
8708 ins_pipe(icond_reg);
8709 %}
8710
8711 // special case for creating a boolean 0 or 1
8712
8713 // n.b. this is selected in preference to the rule above because it
8714 // avoids loading constants 0 and 1 into a source register
8715
8716 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8717 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8718
8719 ins_cost(INSN_COST * 2);
8720 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8721
8722 ins_encode %{
8723 // equivalently
8724 // cset(as_Register($dst$$reg),
8725 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8726 __ csincw(as_Register($dst$$reg),
8727 zr,
8728 zr,
8729 (Assembler::Condition)$cmp$$cmpcode);
8730 %}
8731
8732 ins_pipe(icond_none);
8733 %}
8734
8735 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8736 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8737
8738 ins_cost(INSN_COST * 2);
8739 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8740
8741 ins_encode %{
8742 // equivalently
8743 // cset(as_Register($dst$$reg),
8744 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8745 __ csincw(as_Register($dst$$reg),
8746 zr,
8747 zr,
8748 (Assembler::Condition)$cmp$$cmpcode);
8749 %}
8750
8751 ins_pipe(icond_none);
8752 %}
8753
8754 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8755 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8756
8757 ins_cost(INSN_COST * 2);
8758 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8759
8760 ins_encode %{
8761 __ csel(as_Register($dst$$reg),
8762 as_Register($src2$$reg),
8763 as_Register($src1$$reg),
8764 (Assembler::Condition)$cmp$$cmpcode);
8765 %}
8766
8767 ins_pipe(icond_reg_reg);
8768 %}
8769
8770 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8771 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8772
8773 ins_cost(INSN_COST * 2);
8774 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8775
8776 ins_encode %{
8777 __ csel(as_Register($dst$$reg),
8778 as_Register($src2$$reg),
8779 as_Register($src1$$reg),
8780 (Assembler::Condition)$cmp$$cmpcode);
8781 %}
8782
8783 ins_pipe(icond_reg_reg);
8784 %}
8785
8786 // special cases where one arg is zero
8787
8788 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8789 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8790
8791 ins_cost(INSN_COST * 2);
8792 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8793
8794 ins_encode %{
8795 __ csel(as_Register($dst$$reg),
8796 zr,
8797 as_Register($src$$reg),
8798 (Assembler::Condition)$cmp$$cmpcode);
8799 %}
8800
8801 ins_pipe(icond_reg);
8802 %}
8803
8804 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8805 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8806
8807 ins_cost(INSN_COST * 2);
8808 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8809
8810 ins_encode %{
8811 __ csel(as_Register($dst$$reg),
8812 zr,
8813 as_Register($src$$reg),
8814 (Assembler::Condition)$cmp$$cmpcode);
8815 %}
8816
8817 ins_pipe(icond_reg);
8818 %}
8819
8820 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8821 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8822
8823 ins_cost(INSN_COST * 2);
8824 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8825
8826 ins_encode %{
8827 __ csel(as_Register($dst$$reg),
8828 as_Register($src$$reg),
8829 zr,
8830 (Assembler::Condition)$cmp$$cmpcode);
8831 %}
8832
8833 ins_pipe(icond_reg);
8834 %}
8835
8836 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8837 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8838
8839 ins_cost(INSN_COST * 2);
8840 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8841
8842 ins_encode %{
8843 __ csel(as_Register($dst$$reg),
8844 as_Register($src$$reg),
8845 zr,
8846 (Assembler::Condition)$cmp$$cmpcode);
8847 %}
8848
8849 ins_pipe(icond_reg);
8850 %}
8851
8852 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8853 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8854
8855 ins_cost(INSN_COST * 2);
8856 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8857
8858 ins_encode %{
8859 __ csel(as_Register($dst$$reg),
8860 as_Register($src2$$reg),
8861 as_Register($src1$$reg),
8862 (Assembler::Condition)$cmp$$cmpcode);
8863 %}
8864
8865 ins_pipe(icond_reg_reg);
8866 %}
8867
8868 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8869 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8870
8871 ins_cost(INSN_COST * 2);
8872 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8873
8874 ins_encode %{
8875 __ csel(as_Register($dst$$reg),
8876 as_Register($src2$$reg),
8877 as_Register($src1$$reg),
8878 (Assembler::Condition)$cmp$$cmpcode);
8879 %}
8880
8881 ins_pipe(icond_reg_reg);
8882 %}
8883
8884 // special cases where one arg is zero
8885
8886 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8887 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8888
8889 ins_cost(INSN_COST * 2);
8890 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8891
8892 ins_encode %{
8893 __ csel(as_Register($dst$$reg),
8894 zr,
8895 as_Register($src$$reg),
8896 (Assembler::Condition)$cmp$$cmpcode);
8897 %}
8898
8899 ins_pipe(icond_reg);
8900 %}
8901
8902 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8903 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8904
8905 ins_cost(INSN_COST * 2);
8906 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8907
8908 ins_encode %{
8909 __ csel(as_Register($dst$$reg),
8910 zr,
8911 as_Register($src$$reg),
8912 (Assembler::Condition)$cmp$$cmpcode);
8913 %}
8914
8915 ins_pipe(icond_reg);
8916 %}
8917
8918 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8919 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8920
8921 ins_cost(INSN_COST * 2);
8922 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8923
8924 ins_encode %{
8925 __ csel(as_Register($dst$$reg),
8926 as_Register($src$$reg),
8927 zr,
8928 (Assembler::Condition)$cmp$$cmpcode);
8929 %}
8930
8931 ins_pipe(icond_reg);
8932 %}
8933
8934 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8935 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8936
8937 ins_cost(INSN_COST * 2);
8938 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8939
8940 ins_encode %{
8941 __ csel(as_Register($dst$$reg),
8942 as_Register($src$$reg),
8943 zr,
8944 (Assembler::Condition)$cmp$$cmpcode);
8945 %}
8946
8947 ins_pipe(icond_reg);
8948 %}
8949
8950 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8951 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8952
8953 ins_cost(INSN_COST * 2);
8954 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8955
8956 ins_encode %{
8957 __ cselw(as_Register($dst$$reg),
8958 as_Register($src2$$reg),
8959 as_Register($src1$$reg),
8960 (Assembler::Condition)$cmp$$cmpcode);
8961 %}
8962
8963 ins_pipe(icond_reg_reg);
8964 %}
8965
8966 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8967 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8968
8969 ins_cost(INSN_COST * 2);
8970 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8971
8972 ins_encode %{
8973 __ cselw(as_Register($dst$$reg),
8974 as_Register($src2$$reg),
8975 as_Register($src1$$reg),
8976 (Assembler::Condition)$cmp$$cmpcode);
8977 %}
8978
8979 ins_pipe(icond_reg_reg);
8980 %}
8981
8982 // special cases where one arg is zero
8983
8984 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8985 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8986
8987 ins_cost(INSN_COST * 2);
8988 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8989
8990 ins_encode %{
8991 __ cselw(as_Register($dst$$reg),
8992 zr,
8993 as_Register($src$$reg),
8994 (Assembler::Condition)$cmp$$cmpcode);
8995 %}
8996
8997 ins_pipe(icond_reg);
8998 %}
8999
9000 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9001 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9002
9003 ins_cost(INSN_COST * 2);
9004 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9005
9006 ins_encode %{
9007 __ cselw(as_Register($dst$$reg),
9008 zr,
9009 as_Register($src$$reg),
9010 (Assembler::Condition)$cmp$$cmpcode);
9011 %}
9012
9013 ins_pipe(icond_reg);
9014 %}
9015
9016 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9017 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9018
9019 ins_cost(INSN_COST * 2);
9020 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9021
9022 ins_encode %{
9023 __ cselw(as_Register($dst$$reg),
9024 as_Register($src$$reg),
9025 zr,
9026 (Assembler::Condition)$cmp$$cmpcode);
9027 %}
9028
9029 ins_pipe(icond_reg);
9030 %}
9031
9032 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9033 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9034
9035 ins_cost(INSN_COST * 2);
9036 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9037
9038 ins_encode %{
9039 __ cselw(as_Register($dst$$reg),
9040 as_Register($src$$reg),
9041 zr,
9042 (Assembler::Condition)$cmp$$cmpcode);
9043 %}
9044
9045 ins_pipe(icond_reg);
9046 %}
9047
9048 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9049 %{
9050 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9051
9052 ins_cost(INSN_COST * 3);
9053
9054 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9055 ins_encode %{
9056 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9057 __ fcsels(as_FloatRegister($dst$$reg),
9058 as_FloatRegister($src2$$reg),
9059 as_FloatRegister($src1$$reg),
9060 cond);
9061 %}
9062
9063 ins_pipe(fp_cond_reg_reg_s);
9064 %}
9065
9066 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9067 %{
9068 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9069
9070 ins_cost(INSN_COST * 3);
9071
9072 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9073 ins_encode %{
9074 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9075 __ fcsels(as_FloatRegister($dst$$reg),
9076 as_FloatRegister($src2$$reg),
9077 as_FloatRegister($src1$$reg),
9078 cond);
9079 %}
9080
9081 ins_pipe(fp_cond_reg_reg_s);
9082 %}
9083
9084 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9085 %{
9086 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9087
9088 ins_cost(INSN_COST * 3);
9089
9090 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9091 ins_encode %{
9092 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9093 __ fcseld(as_FloatRegister($dst$$reg),
9094 as_FloatRegister($src2$$reg),
9095 as_FloatRegister($src1$$reg),
9096 cond);
9097 %}
9098
9099 ins_pipe(fp_cond_reg_reg_d);
9100 %}
9101
9102 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9103 %{
9104 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9105
9106 ins_cost(INSN_COST * 3);
9107
9108 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9109 ins_encode %{
9110 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9111 __ fcseld(as_FloatRegister($dst$$reg),
9112 as_FloatRegister($src2$$reg),
9113 as_FloatRegister($src1$$reg),
9114 cond);
9115 %}
9116
9117 ins_pipe(fp_cond_reg_reg_d);
9118 %}
9119
9120 // ============================================================================
9121 // Arithmetic Instructions
9122 //
9123
9124 // Integer Addition
9125
9126 // TODO
9127 // these currently employ operations which do not set CR and hence are
9128 // not flagged as killing CR but we would like to isolate the cases
9129 // where we want to set flags from those where we don't. need to work
9130 // out how to do that.
9131
9132 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9133 match(Set dst (AddI src1 src2));
9134
9135 ins_cost(INSN_COST);
9136 format %{ "addw $dst, $src1, $src2" %}
9137
9138 ins_encode %{
9139 __ addw(as_Register($dst$$reg),
9140 as_Register($src1$$reg),
9141 as_Register($src2$$reg));
9142 %}
9143
9144 ins_pipe(ialu_reg_reg);
9145 %}
9146
9147 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9148 match(Set dst (AddI src1 src2));
9149
9150 ins_cost(INSN_COST);
9151 format %{ "addw $dst, $src1, $src2" %}
9152
9153 // use opcode to indicate that this is an add not a sub
9154 opcode(0x0);
9155
9156 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9157
9158 ins_pipe(ialu_reg_imm);
9159 %}
9160
9161 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9162 match(Set dst (AddI (ConvL2I src1) src2));
9163
9164 ins_cost(INSN_COST);
9165 format %{ "addw $dst, $src1, $src2" %}
9166
9167 // use opcode to indicate that this is an add not a sub
9168 opcode(0x0);
9169
9170 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9171
9172 ins_pipe(ialu_reg_imm);
9173 %}
9174
9175 // Pointer Addition
9176 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9177 match(Set dst (AddP src1 src2));
9178
9179 ins_cost(INSN_COST);
9180 format %{ "add $dst, $src1, $src2\t# ptr" %}
9181
9182 ins_encode %{
9183 __ add(as_Register($dst$$reg),
9184 as_Register($src1$$reg),
9185 as_Register($src2$$reg));
9186 %}
9187
9188 ins_pipe(ialu_reg_reg);
9189 %}
9190
9191 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9192 match(Set dst (AddP src1 (ConvI2L src2)));
9193
9194 ins_cost(1.9 * INSN_COST);
9195 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9196
9197 ins_encode %{
9198 __ add(as_Register($dst$$reg),
9199 as_Register($src1$$reg),
9200 as_Register($src2$$reg), ext::sxtw);
9201 %}
9202
9203 ins_pipe(ialu_reg_reg);
9204 %}
9205
9206 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9207 match(Set dst (AddP src1 (LShiftL src2 scale)));
9208
9209 ins_cost(1.9 * INSN_COST);
9210 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9211
9212 ins_encode %{
9213 __ lea(as_Register($dst$$reg),
9214 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9215 Address::lsl($scale$$constant)));
9216 %}
9217
9218 ins_pipe(ialu_reg_reg_shift);
9219 %}
9220
9221 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9222 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9223
9224 ins_cost(1.9 * INSN_COST);
9225 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9226
9227 ins_encode %{
9228 __ lea(as_Register($dst$$reg),
9229 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9230 Address::sxtw($scale$$constant)));
9231 %}
9232
9233 ins_pipe(ialu_reg_reg_shift);
9234 %}
9235
9236 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9237 match(Set dst (LShiftL (ConvI2L src) scale));
9238
9239 ins_cost(INSN_COST);
9240 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9241
9242 ins_encode %{
9243 __ sbfiz(as_Register($dst$$reg),
9244 as_Register($src$$reg),
9245 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9246 %}
9247
9248 ins_pipe(ialu_reg_shift);
9249 %}
9250
9251 // Pointer Immediate Addition
9252 // n.b. this needs to be more expensive than using an indirect memory
9253 // operand
9254 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9255 match(Set dst (AddP src1 src2));
9256
9257 ins_cost(INSN_COST);
9258 format %{ "add $dst, $src1, $src2\t# ptr" %}
9259
9260 // use opcode to indicate that this is an add not a sub
9261 opcode(0x0);
9262
9263 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9264
9265 ins_pipe(ialu_reg_imm);
9266 %}
9267
9268 // Long Addition
9269 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9270
9271 match(Set dst (AddL src1 src2));
9272
9273 ins_cost(INSN_COST);
9274 format %{ "add $dst, $src1, $src2" %}
9275
9276 ins_encode %{
9277 __ add(as_Register($dst$$reg),
9278 as_Register($src1$$reg),
9279 as_Register($src2$$reg));
9280 %}
9281
9282 ins_pipe(ialu_reg_reg);
9283 %}
9284
9285 // No constant pool entries requiredLong Immediate Addition.
9286 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9287 match(Set dst (AddL src1 src2));
9288
9289 ins_cost(INSN_COST);
9290 format %{ "add $dst, $src1, $src2" %}
9291
9292 // use opcode to indicate that this is an add not a sub
9293 opcode(0x0);
9294
9295 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9296
9297 ins_pipe(ialu_reg_imm);
9298 %}
9299
9300 // Integer Subtraction
9301 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9302 match(Set dst (SubI src1 src2));
9303
9304 ins_cost(INSN_COST);
9305 format %{ "subw $dst, $src1, $src2" %}
9306
9307 ins_encode %{
9308 __ subw(as_Register($dst$$reg),
9309 as_Register($src1$$reg),
9310 as_Register($src2$$reg));
9311 %}
9312
9313 ins_pipe(ialu_reg_reg);
9314 %}
9315
9316 // Immediate Subtraction
9317 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9318 match(Set dst (SubI src1 src2));
9319
9320 ins_cost(INSN_COST);
9321 format %{ "subw $dst, $src1, $src2" %}
9322
9323 // use opcode to indicate that this is a sub not an add
9324 opcode(0x1);
9325
9326 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9327
9328 ins_pipe(ialu_reg_imm);
9329 %}
9330
9331 // Long Subtraction
9332 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9333
9334 match(Set dst (SubL src1 src2));
9335
9336 ins_cost(INSN_COST);
9337 format %{ "sub $dst, $src1, $src2" %}
9338
9339 ins_encode %{
9340 __ sub(as_Register($dst$$reg),
9341 as_Register($src1$$reg),
9342 as_Register($src2$$reg));
9343 %}
9344
9345 ins_pipe(ialu_reg_reg);
9346 %}
9347
9348 // No constant pool entries requiredLong Immediate Subtraction.
9349 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9350 match(Set dst (SubL src1 src2));
9351
9352 ins_cost(INSN_COST);
9353 format %{ "sub$dst, $src1, $src2" %}
9354
9355 // use opcode to indicate that this is a sub not an add
9356 opcode(0x1);
9357
9358 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9359
9360 ins_pipe(ialu_reg_imm);
9361 %}
9362
9363 // Integer Negation (special case for sub)
9364
9365 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9366 match(Set dst (SubI zero src));
9367
9368 ins_cost(INSN_COST);
9369 format %{ "negw $dst, $src\t# int" %}
9370
9371 ins_encode %{
9372 __ negw(as_Register($dst$$reg),
9373 as_Register($src$$reg));
9374 %}
9375
9376 ins_pipe(ialu_reg);
9377 %}
9378
9379 // Long Negation
9380
9381 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9382 match(Set dst (SubL zero src));
9383
9384 ins_cost(INSN_COST);
9385 format %{ "neg $dst, $src\t# long" %}
9386
9387 ins_encode %{
9388 __ neg(as_Register($dst$$reg),
9389 as_Register($src$$reg));
9390 %}
9391
9392 ins_pipe(ialu_reg);
9393 %}
9394
9395 // Integer Multiply
9396
9397 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9398 match(Set dst (MulI src1 src2));
9399
9400 ins_cost(INSN_COST * 3);
9401 format %{ "mulw $dst, $src1, $src2" %}
9402
9403 ins_encode %{
9404 __ mulw(as_Register($dst$$reg),
9405 as_Register($src1$$reg),
9406 as_Register($src2$$reg));
9407 %}
9408
9409 ins_pipe(imul_reg_reg);
9410 %}
9411
9412 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9413 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9414
9415 ins_cost(INSN_COST * 3);
9416 format %{ "smull $dst, $src1, $src2" %}
9417
9418 ins_encode %{
9419 __ smull(as_Register($dst$$reg),
9420 as_Register($src1$$reg),
9421 as_Register($src2$$reg));
9422 %}
9423
9424 ins_pipe(imul_reg_reg);
9425 %}
9426
9427 // Long Multiply
9428
9429 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9430 match(Set dst (MulL src1 src2));
9431
9432 ins_cost(INSN_COST * 5);
9433 format %{ "mul $dst, $src1, $src2" %}
9434
9435 ins_encode %{
9436 __ mul(as_Register($dst$$reg),
9437 as_Register($src1$$reg),
9438 as_Register($src2$$reg));
9439 %}
9440
9441 ins_pipe(lmul_reg_reg);
9442 %}
9443
9444 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9445 %{
9446 match(Set dst (MulHiL src1 src2));
9447
9448 ins_cost(INSN_COST * 7);
9449 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9450
9451 ins_encode %{
9452 __ smulh(as_Register($dst$$reg),
9453 as_Register($src1$$reg),
9454 as_Register($src2$$reg));
9455 %}
9456
9457 ins_pipe(lmul_reg_reg);
9458 %}
9459
9460 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9461 %{
9462 match(Set dst (UMulHiL src1 src2));
9463
9464 ins_cost(INSN_COST * 7);
9465 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9466
9467 ins_encode %{
9468 __ umulh(as_Register($dst$$reg),
9469 as_Register($src1$$reg),
9470 as_Register($src2$$reg));
9471 %}
9472
9473 ins_pipe(lmul_reg_reg);
9474 %}
9475
9476 // Combined Integer Multiply & Add/Sub
9477
9478 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9479 match(Set dst (AddI src3 (MulI src1 src2)));
9480
9481 ins_cost(INSN_COST * 3);
9482 format %{ "madd $dst, $src1, $src2, $src3" %}
9483
9484 ins_encode %{
9485 __ maddw(as_Register($dst$$reg),
9486 as_Register($src1$$reg),
9487 as_Register($src2$$reg),
9488 as_Register($src3$$reg));
9489 %}
9490
9491 ins_pipe(imac_reg_reg);
9492 %}
9493
9494 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9495 match(Set dst (SubI src3 (MulI src1 src2)));
9496
9497 ins_cost(INSN_COST * 3);
9498 format %{ "msub $dst, $src1, $src2, $src3" %}
9499
9500 ins_encode %{
9501 __ msubw(as_Register($dst$$reg),
9502 as_Register($src1$$reg),
9503 as_Register($src2$$reg),
9504 as_Register($src3$$reg));
9505 %}
9506
9507 ins_pipe(imac_reg_reg);
9508 %}
9509
9510 // Combined Integer Multiply & Neg
9511
9512 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9513 match(Set dst (MulI (SubI zero src1) src2));
9514
9515 ins_cost(INSN_COST * 3);
9516 format %{ "mneg $dst, $src1, $src2" %}
9517
9518 ins_encode %{
9519 __ mnegw(as_Register($dst$$reg),
9520 as_Register($src1$$reg),
9521 as_Register($src2$$reg));
9522 %}
9523
9524 ins_pipe(imac_reg_reg);
9525 %}
9526
9527 // Combined Long Multiply & Add/Sub
9528
9529 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9530 match(Set dst (AddL src3 (MulL src1 src2)));
9531
9532 ins_cost(INSN_COST * 5);
9533 format %{ "madd $dst, $src1, $src2, $src3" %}
9534
9535 ins_encode %{
9536 __ madd(as_Register($dst$$reg),
9537 as_Register($src1$$reg),
9538 as_Register($src2$$reg),
9539 as_Register($src3$$reg));
9540 %}
9541
9542 ins_pipe(lmac_reg_reg);
9543 %}
9544
9545 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9546 match(Set dst (SubL src3 (MulL src1 src2)));
9547
9548 ins_cost(INSN_COST * 5);
9549 format %{ "msub $dst, $src1, $src2, $src3" %}
9550
9551 ins_encode %{
9552 __ msub(as_Register($dst$$reg),
9553 as_Register($src1$$reg),
9554 as_Register($src2$$reg),
9555 as_Register($src3$$reg));
9556 %}
9557
9558 ins_pipe(lmac_reg_reg);
9559 %}
9560
9561 // Combined Long Multiply & Neg
9562
9563 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9564 match(Set dst (MulL (SubL zero src1) src2));
9565
9566 ins_cost(INSN_COST * 5);
9567 format %{ "mneg $dst, $src1, $src2" %}
9568
9569 ins_encode %{
9570 __ mneg(as_Register($dst$$reg),
9571 as_Register($src1$$reg),
9572 as_Register($src2$$reg));
9573 %}
9574
9575 ins_pipe(lmac_reg_reg);
9576 %}
9577
9578 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9579
9580 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9581 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9582
9583 ins_cost(INSN_COST * 3);
9584 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9585
9586 ins_encode %{
9587 __ smaddl(as_Register($dst$$reg),
9588 as_Register($src1$$reg),
9589 as_Register($src2$$reg),
9590 as_Register($src3$$reg));
9591 %}
9592
9593 ins_pipe(imac_reg_reg);
9594 %}
9595
9596 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9597 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9598
9599 ins_cost(INSN_COST * 3);
9600 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9601
9602 ins_encode %{
9603 __ smsubl(as_Register($dst$$reg),
9604 as_Register($src1$$reg),
9605 as_Register($src2$$reg),
9606 as_Register($src3$$reg));
9607 %}
9608
9609 ins_pipe(imac_reg_reg);
9610 %}
9611
9612 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9613 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9614
9615 ins_cost(INSN_COST * 3);
9616 format %{ "smnegl $dst, $src1, $src2" %}
9617
9618 ins_encode %{
9619 __ smnegl(as_Register($dst$$reg),
9620 as_Register($src1$$reg),
9621 as_Register($src2$$reg));
9622 %}
9623
9624 ins_pipe(imac_reg_reg);
9625 %}
9626
9627 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9628
9629 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9630 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9631
9632 ins_cost(INSN_COST * 5);
9633 format %{ "mulw rscratch1, $src1, $src2\n\t"
9634 "maddw $dst, $src3, $src4, rscratch1" %}
9635
9636 ins_encode %{
9637 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9638 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9639
9640 ins_pipe(imac_reg_reg);
9641 %}
9642
9643 // Integer Divide
9644
9645 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9646 match(Set dst (DivI src1 src2));
9647
9648 ins_cost(INSN_COST * 19);
9649 format %{ "sdivw $dst, $src1, $src2" %}
9650
9651 ins_encode(aarch64_enc_divw(dst, src1, src2));
9652 ins_pipe(idiv_reg_reg);
9653 %}
9654
9655 // Long Divide
9656
9657 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9658 match(Set dst (DivL src1 src2));
9659
9660 ins_cost(INSN_COST * 35);
9661 format %{ "sdiv $dst, $src1, $src2" %}
9662
9663 ins_encode(aarch64_enc_div(dst, src1, src2));
9664 ins_pipe(ldiv_reg_reg);
9665 %}
9666
9667 // Integer Remainder
9668
9669 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9670 match(Set dst (ModI src1 src2));
9671
9672 ins_cost(INSN_COST * 22);
9673 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9674 "msubw $dst, rscratch1, $src2, $src1" %}
9675
9676 ins_encode(aarch64_enc_modw(dst, src1, src2));
9677 ins_pipe(idiv_reg_reg);
9678 %}
9679
9680 // Long Remainder
9681
9682 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9683 match(Set dst (ModL src1 src2));
9684
9685 ins_cost(INSN_COST * 38);
9686 format %{ "sdiv rscratch1, $src1, $src2\n"
9687 "msub $dst, rscratch1, $src2, $src1" %}
9688
9689 ins_encode(aarch64_enc_mod(dst, src1, src2));
9690 ins_pipe(ldiv_reg_reg);
9691 %}
9692
9693 // Unsigned Integer Divide
9694
9695 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9696 match(Set dst (UDivI src1 src2));
9697
9698 ins_cost(INSN_COST * 19);
9699 format %{ "udivw $dst, $src1, $src2" %}
9700
9701 ins_encode %{
9702 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9703 %}
9704
9705 ins_pipe(idiv_reg_reg);
9706 %}
9707
9708 // Unsigned Long Divide
9709
9710 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9711 match(Set dst (UDivL src1 src2));
9712
9713 ins_cost(INSN_COST * 35);
9714 format %{ "udiv $dst, $src1, $src2" %}
9715
9716 ins_encode %{
9717 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9718 %}
9719
9720 ins_pipe(ldiv_reg_reg);
9721 %}
9722
9723 // Unsigned Integer Remainder
9724
9725 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9726 match(Set dst (UModI src1 src2));
9727
9728 ins_cost(INSN_COST * 22);
9729 format %{ "udivw rscratch1, $src1, $src2\n\t"
9730 "msubw $dst, rscratch1, $src2, $src1" %}
9731
9732 ins_encode %{
9733 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9734 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9735 %}
9736
9737 ins_pipe(idiv_reg_reg);
9738 %}
9739
9740 // Unsigned Long Remainder
9741
9742 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9743 match(Set dst (UModL src1 src2));
9744
9745 ins_cost(INSN_COST * 38);
9746 format %{ "udiv rscratch1, $src1, $src2\n"
9747 "msub $dst, rscratch1, $src2, $src1" %}
9748
9749 ins_encode %{
9750 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9751 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9752 %}
9753
9754 ins_pipe(ldiv_reg_reg);
9755 %}
9756
9757 // Integer Shifts
9758
9759 // Shift Left Register
9760 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9761 match(Set dst (LShiftI src1 src2));
9762
9763 ins_cost(INSN_COST * 2);
9764 format %{ "lslvw $dst, $src1, $src2" %}
9765
9766 ins_encode %{
9767 __ lslvw(as_Register($dst$$reg),
9768 as_Register($src1$$reg),
9769 as_Register($src2$$reg));
9770 %}
9771
9772 ins_pipe(ialu_reg_reg_vshift);
9773 %}
9774
9775 // Shift Left Immediate
9776 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9777 match(Set dst (LShiftI src1 src2));
9778
9779 ins_cost(INSN_COST);
9780 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9781
9782 ins_encode %{
9783 __ lslw(as_Register($dst$$reg),
9784 as_Register($src1$$reg),
9785 $src2$$constant & 0x1f);
9786 %}
9787
9788 ins_pipe(ialu_reg_shift);
9789 %}
9790
9791 // Shift Right Logical Register
9792 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9793 match(Set dst (URShiftI src1 src2));
9794
9795 ins_cost(INSN_COST * 2);
9796 format %{ "lsrvw $dst, $src1, $src2" %}
9797
9798 ins_encode %{
9799 __ lsrvw(as_Register($dst$$reg),
9800 as_Register($src1$$reg),
9801 as_Register($src2$$reg));
9802 %}
9803
9804 ins_pipe(ialu_reg_reg_vshift);
9805 %}
9806
9807 // Shift Right Logical Immediate
9808 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9809 match(Set dst (URShiftI src1 src2));
9810
9811 ins_cost(INSN_COST);
9812 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9813
9814 ins_encode %{
9815 __ lsrw(as_Register($dst$$reg),
9816 as_Register($src1$$reg),
9817 $src2$$constant & 0x1f);
9818 %}
9819
9820 ins_pipe(ialu_reg_shift);
9821 %}
9822
9823 // Shift Right Arithmetic Register
9824 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9825 match(Set dst (RShiftI src1 src2));
9826
9827 ins_cost(INSN_COST * 2);
9828 format %{ "asrvw $dst, $src1, $src2" %}
9829
9830 ins_encode %{
9831 __ asrvw(as_Register($dst$$reg),
9832 as_Register($src1$$reg),
9833 as_Register($src2$$reg));
9834 %}
9835
9836 ins_pipe(ialu_reg_reg_vshift);
9837 %}
9838
9839 // Shift Right Arithmetic Immediate
9840 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9841 match(Set dst (RShiftI src1 src2));
9842
9843 ins_cost(INSN_COST);
9844 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9845
9846 ins_encode %{
9847 __ asrw(as_Register($dst$$reg),
9848 as_Register($src1$$reg),
9849 $src2$$constant & 0x1f);
9850 %}
9851
9852 ins_pipe(ialu_reg_shift);
9853 %}
9854
9855 // Combined Int Mask and Right Shift (using UBFM)
9856 // TODO
9857
9858 // Long Shifts
9859
9860 // Shift Left Register
9861 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9862 match(Set dst (LShiftL src1 src2));
9863
9864 ins_cost(INSN_COST * 2);
9865 format %{ "lslv $dst, $src1, $src2" %}
9866
9867 ins_encode %{
9868 __ lslv(as_Register($dst$$reg),
9869 as_Register($src1$$reg),
9870 as_Register($src2$$reg));
9871 %}
9872
9873 ins_pipe(ialu_reg_reg_vshift);
9874 %}
9875
9876 // Shift Left Immediate
9877 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9878 match(Set dst (LShiftL src1 src2));
9879
9880 ins_cost(INSN_COST);
9881 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9882
9883 ins_encode %{
9884 __ lsl(as_Register($dst$$reg),
9885 as_Register($src1$$reg),
9886 $src2$$constant & 0x3f);
9887 %}
9888
9889 ins_pipe(ialu_reg_shift);
9890 %}
9891
9892 // Shift Right Logical Register
9893 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9894 match(Set dst (URShiftL src1 src2));
9895
9896 ins_cost(INSN_COST * 2);
9897 format %{ "lsrv $dst, $src1, $src2" %}
9898
9899 ins_encode %{
9900 __ lsrv(as_Register($dst$$reg),
9901 as_Register($src1$$reg),
9902 as_Register($src2$$reg));
9903 %}
9904
9905 ins_pipe(ialu_reg_reg_vshift);
9906 %}
9907
9908 // Shift Right Logical Immediate
9909 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9910 match(Set dst (URShiftL src1 src2));
9911
9912 ins_cost(INSN_COST);
9913 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9914
9915 ins_encode %{
9916 __ lsr(as_Register($dst$$reg),
9917 as_Register($src1$$reg),
9918 $src2$$constant & 0x3f);
9919 %}
9920
9921 ins_pipe(ialu_reg_shift);
9922 %}
9923
9924 // A special-case pattern for card table stores.
9925 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9926 match(Set dst (URShiftL (CastP2X src1) src2));
9927
9928 ins_cost(INSN_COST);
9929 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9930
9931 ins_encode %{
9932 __ lsr(as_Register($dst$$reg),
9933 as_Register($src1$$reg),
9934 $src2$$constant & 0x3f);
9935 %}
9936
9937 ins_pipe(ialu_reg_shift);
9938 %}
9939
9940 // Shift Right Arithmetic Register
9941 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9942 match(Set dst (RShiftL src1 src2));
9943
9944 ins_cost(INSN_COST * 2);
9945 format %{ "asrv $dst, $src1, $src2" %}
9946
9947 ins_encode %{
9948 __ asrv(as_Register($dst$$reg),
9949 as_Register($src1$$reg),
9950 as_Register($src2$$reg));
9951 %}
9952
9953 ins_pipe(ialu_reg_reg_vshift);
9954 %}
9955
9956 // Shift Right Arithmetic Immediate
9957 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9958 match(Set dst (RShiftL src1 src2));
9959
9960 ins_cost(INSN_COST);
9961 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9962
9963 ins_encode %{
9964 __ asr(as_Register($dst$$reg),
9965 as_Register($src1$$reg),
9966 $src2$$constant & 0x3f);
9967 %}
9968
9969 ins_pipe(ialu_reg_shift);
9970 %}
9971
9972 // BEGIN This section of the file is automatically generated. Do not edit --------------
9973 // This section is generated from aarch64_ad.m4
9974
9975 // This pattern is automatically generated from aarch64_ad.m4
9976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9977 instruct regL_not_reg(iRegLNoSp dst,
9978 iRegL src1, immL_M1 m1,
9979 rFlagsReg cr) %{
9980 match(Set dst (XorL src1 m1));
9981 ins_cost(INSN_COST);
9982 format %{ "eon $dst, $src1, zr" %}
9983
9984 ins_encode %{
9985 __ eon(as_Register($dst$$reg),
9986 as_Register($src1$$reg),
9987 zr,
9988 Assembler::LSL, 0);
9989 %}
9990
9991 ins_pipe(ialu_reg);
9992 %}
9993
9994 // This pattern is automatically generated from aarch64_ad.m4
9995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9996 instruct regI_not_reg(iRegINoSp dst,
9997 iRegIorL2I src1, immI_M1 m1,
9998 rFlagsReg cr) %{
9999 match(Set dst (XorI src1 m1));
10000 ins_cost(INSN_COST);
10001 format %{ "eonw $dst, $src1, zr" %}
10002
10003 ins_encode %{
10004 __ eonw(as_Register($dst$$reg),
10005 as_Register($src1$$reg),
10006 zr,
10007 Assembler::LSL, 0);
10008 %}
10009
10010 ins_pipe(ialu_reg);
10011 %}
10012
10013 // This pattern is automatically generated from aarch64_ad.m4
10014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10015 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10016 immI0 zero, iRegIorL2I src1, immI src2) %{
10017 match(Set dst (SubI zero (URShiftI src1 src2)));
10018
10019 ins_cost(1.9 * INSN_COST);
10020 format %{ "negw $dst, $src1, LSR $src2" %}
10021
10022 ins_encode %{
10023 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10024 Assembler::LSR, $src2$$constant & 0x1f);
10025 %}
10026
10027 ins_pipe(ialu_reg_shift);
10028 %}
10029
10030 // This pattern is automatically generated from aarch64_ad.m4
10031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10032 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10033 immI0 zero, iRegIorL2I src1, immI src2) %{
10034 match(Set dst (SubI zero (RShiftI src1 src2)));
10035
10036 ins_cost(1.9 * INSN_COST);
10037 format %{ "negw $dst, $src1, ASR $src2" %}
10038
10039 ins_encode %{
10040 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10041 Assembler::ASR, $src2$$constant & 0x1f);
10042 %}
10043
10044 ins_pipe(ialu_reg_shift);
10045 %}
10046
10047 // This pattern is automatically generated from aarch64_ad.m4
10048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10049 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10050 immI0 zero, iRegIorL2I src1, immI src2) %{
10051 match(Set dst (SubI zero (LShiftI src1 src2)));
10052
10053 ins_cost(1.9 * INSN_COST);
10054 format %{ "negw $dst, $src1, LSL $src2" %}
10055
10056 ins_encode %{
10057 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10058 Assembler::LSL, $src2$$constant & 0x1f);
10059 %}
10060
10061 ins_pipe(ialu_reg_shift);
10062 %}
10063
10064 // This pattern is automatically generated from aarch64_ad.m4
10065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10066 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10067 immL0 zero, iRegL src1, immI src2) %{
10068 match(Set dst (SubL zero (URShiftL src1 src2)));
10069
10070 ins_cost(1.9 * INSN_COST);
10071 format %{ "neg $dst, $src1, LSR $src2" %}
10072
10073 ins_encode %{
10074 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10075 Assembler::LSR, $src2$$constant & 0x3f);
10076 %}
10077
10078 ins_pipe(ialu_reg_shift);
10079 %}
10080
10081 // This pattern is automatically generated from aarch64_ad.m4
10082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10083 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10084 immL0 zero, iRegL src1, immI src2) %{
10085 match(Set dst (SubL zero (RShiftL src1 src2)));
10086
10087 ins_cost(1.9 * INSN_COST);
10088 format %{ "neg $dst, $src1, ASR $src2" %}
10089
10090 ins_encode %{
10091 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10092 Assembler::ASR, $src2$$constant & 0x3f);
10093 %}
10094
10095 ins_pipe(ialu_reg_shift);
10096 %}
10097
10098 // This pattern is automatically generated from aarch64_ad.m4
10099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10100 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10101 immL0 zero, iRegL src1, immI src2) %{
10102 match(Set dst (SubL zero (LShiftL src1 src2)));
10103
10104 ins_cost(1.9 * INSN_COST);
10105 format %{ "neg $dst, $src1, LSL $src2" %}
10106
10107 ins_encode %{
10108 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10109 Assembler::LSL, $src2$$constant & 0x3f);
10110 %}
10111
10112 ins_pipe(ialu_reg_shift);
10113 %}
10114
10115 // This pattern is automatically generated from aarch64_ad.m4
10116 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10117 instruct AndI_reg_not_reg(iRegINoSp dst,
10118 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10119 match(Set dst (AndI src1 (XorI src2 m1)));
10120 ins_cost(INSN_COST);
10121 format %{ "bicw $dst, $src1, $src2" %}
10122
10123 ins_encode %{
10124 __ bicw(as_Register($dst$$reg),
10125 as_Register($src1$$reg),
10126 as_Register($src2$$reg),
10127 Assembler::LSL, 0);
10128 %}
10129
10130 ins_pipe(ialu_reg_reg);
10131 %}
10132
10133 // This pattern is automatically generated from aarch64_ad.m4
10134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10135 instruct AndL_reg_not_reg(iRegLNoSp dst,
10136 iRegL src1, iRegL src2, immL_M1 m1) %{
10137 match(Set dst (AndL src1 (XorL src2 m1)));
10138 ins_cost(INSN_COST);
10139 format %{ "bic $dst, $src1, $src2" %}
10140
10141 ins_encode %{
10142 __ bic(as_Register($dst$$reg),
10143 as_Register($src1$$reg),
10144 as_Register($src2$$reg),
10145 Assembler::LSL, 0);
10146 %}
10147
10148 ins_pipe(ialu_reg_reg);
10149 %}
10150
10151 // This pattern is automatically generated from aarch64_ad.m4
10152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10153 instruct OrI_reg_not_reg(iRegINoSp dst,
10154 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10155 match(Set dst (OrI src1 (XorI src2 m1)));
10156 ins_cost(INSN_COST);
10157 format %{ "ornw $dst, $src1, $src2" %}
10158
10159 ins_encode %{
10160 __ ornw(as_Register($dst$$reg),
10161 as_Register($src1$$reg),
10162 as_Register($src2$$reg),
10163 Assembler::LSL, 0);
10164 %}
10165
10166 ins_pipe(ialu_reg_reg);
10167 %}
10168
10169 // This pattern is automatically generated from aarch64_ad.m4
10170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10171 instruct OrL_reg_not_reg(iRegLNoSp dst,
10172 iRegL src1, iRegL src2, immL_M1 m1) %{
10173 match(Set dst (OrL src1 (XorL src2 m1)));
10174 ins_cost(INSN_COST);
10175 format %{ "orn $dst, $src1, $src2" %}
10176
10177 ins_encode %{
10178 __ orn(as_Register($dst$$reg),
10179 as_Register($src1$$reg),
10180 as_Register($src2$$reg),
10181 Assembler::LSL, 0);
10182 %}
10183
10184 ins_pipe(ialu_reg_reg);
10185 %}
10186
10187 // This pattern is automatically generated from aarch64_ad.m4
10188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10189 instruct XorI_reg_not_reg(iRegINoSp dst,
10190 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10191 match(Set dst (XorI m1 (XorI src2 src1)));
10192 ins_cost(INSN_COST);
10193 format %{ "eonw $dst, $src1, $src2" %}
10194
10195 ins_encode %{
10196 __ eonw(as_Register($dst$$reg),
10197 as_Register($src1$$reg),
10198 as_Register($src2$$reg),
10199 Assembler::LSL, 0);
10200 %}
10201
10202 ins_pipe(ialu_reg_reg);
10203 %}
10204
10205 // This pattern is automatically generated from aarch64_ad.m4
10206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10207 instruct XorL_reg_not_reg(iRegLNoSp dst,
10208 iRegL src1, iRegL src2, immL_M1 m1) %{
10209 match(Set dst (XorL m1 (XorL src2 src1)));
10210 ins_cost(INSN_COST);
10211 format %{ "eon $dst, $src1, $src2" %}
10212
10213 ins_encode %{
10214 __ eon(as_Register($dst$$reg),
10215 as_Register($src1$$reg),
10216 as_Register($src2$$reg),
10217 Assembler::LSL, 0);
10218 %}
10219
10220 ins_pipe(ialu_reg_reg);
10221 %}
10222
10223 // This pattern is automatically generated from aarch64_ad.m4
10224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10225 // val & (-1 ^ (val >>> shift)) ==> bicw
10226 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10227 iRegIorL2I src1, iRegIorL2I src2,
10228 immI src3, immI_M1 src4) %{
10229 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10230 ins_cost(1.9 * INSN_COST);
10231 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10232
10233 ins_encode %{
10234 __ bicw(as_Register($dst$$reg),
10235 as_Register($src1$$reg),
10236 as_Register($src2$$reg),
10237 Assembler::LSR,
10238 $src3$$constant & 0x1f);
10239 %}
10240
10241 ins_pipe(ialu_reg_reg_shift);
10242 %}
10243
10244 // This pattern is automatically generated from aarch64_ad.m4
10245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10246 // val & (-1 ^ (val >>> shift)) ==> bic
10247 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10248 iRegL src1, iRegL src2,
10249 immI src3, immL_M1 src4) %{
10250 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10251 ins_cost(1.9 * INSN_COST);
10252 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10253
10254 ins_encode %{
10255 __ bic(as_Register($dst$$reg),
10256 as_Register($src1$$reg),
10257 as_Register($src2$$reg),
10258 Assembler::LSR,
10259 $src3$$constant & 0x3f);
10260 %}
10261
10262 ins_pipe(ialu_reg_reg_shift);
10263 %}
10264
10265 // This pattern is automatically generated from aarch64_ad.m4
10266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10267 // val & (-1 ^ (val >> shift)) ==> bicw
10268 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10269 iRegIorL2I src1, iRegIorL2I src2,
10270 immI src3, immI_M1 src4) %{
10271 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10272 ins_cost(1.9 * INSN_COST);
10273 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10274
10275 ins_encode %{
10276 __ bicw(as_Register($dst$$reg),
10277 as_Register($src1$$reg),
10278 as_Register($src2$$reg),
10279 Assembler::ASR,
10280 $src3$$constant & 0x1f);
10281 %}
10282
10283 ins_pipe(ialu_reg_reg_shift);
10284 %}
10285
10286 // This pattern is automatically generated from aarch64_ad.m4
10287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10288 // val & (-1 ^ (val >> shift)) ==> bic
10289 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10290 iRegL src1, iRegL src2,
10291 immI src3, immL_M1 src4) %{
10292 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10293 ins_cost(1.9 * INSN_COST);
10294 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10295
10296 ins_encode %{
10297 __ bic(as_Register($dst$$reg),
10298 as_Register($src1$$reg),
10299 as_Register($src2$$reg),
10300 Assembler::ASR,
10301 $src3$$constant & 0x3f);
10302 %}
10303
10304 ins_pipe(ialu_reg_reg_shift);
10305 %}
10306
10307 // This pattern is automatically generated from aarch64_ad.m4
10308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10309 // val & (-1 ^ (val ror shift)) ==> bicw
10310 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10311 iRegIorL2I src1, iRegIorL2I src2,
10312 immI src3, immI_M1 src4) %{
10313 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10314 ins_cost(1.9 * INSN_COST);
10315 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10316
10317 ins_encode %{
10318 __ bicw(as_Register($dst$$reg),
10319 as_Register($src1$$reg),
10320 as_Register($src2$$reg),
10321 Assembler::ROR,
10322 $src3$$constant & 0x1f);
10323 %}
10324
10325 ins_pipe(ialu_reg_reg_shift);
10326 %}
10327
10328 // This pattern is automatically generated from aarch64_ad.m4
10329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10330 // val & (-1 ^ (val ror shift)) ==> bic
10331 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10332 iRegL src1, iRegL src2,
10333 immI src3, immL_M1 src4) %{
10334 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10335 ins_cost(1.9 * INSN_COST);
10336 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10337
10338 ins_encode %{
10339 __ bic(as_Register($dst$$reg),
10340 as_Register($src1$$reg),
10341 as_Register($src2$$reg),
10342 Assembler::ROR,
10343 $src3$$constant & 0x3f);
10344 %}
10345
10346 ins_pipe(ialu_reg_reg_shift);
10347 %}
10348
10349 // This pattern is automatically generated from aarch64_ad.m4
10350 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10351 // val & (-1 ^ (val << shift)) ==> bicw
10352 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10353 iRegIorL2I src1, iRegIorL2I src2,
10354 immI src3, immI_M1 src4) %{
10355 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10356 ins_cost(1.9 * INSN_COST);
10357 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10358
10359 ins_encode %{
10360 __ bicw(as_Register($dst$$reg),
10361 as_Register($src1$$reg),
10362 as_Register($src2$$reg),
10363 Assembler::LSL,
10364 $src3$$constant & 0x1f);
10365 %}
10366
10367 ins_pipe(ialu_reg_reg_shift);
10368 %}
10369
10370 // This pattern is automatically generated from aarch64_ad.m4
10371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10372 // val & (-1 ^ (val << shift)) ==> bic
10373 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10374 iRegL src1, iRegL src2,
10375 immI src3, immL_M1 src4) %{
10376 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10377 ins_cost(1.9 * INSN_COST);
10378 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10379
10380 ins_encode %{
10381 __ bic(as_Register($dst$$reg),
10382 as_Register($src1$$reg),
10383 as_Register($src2$$reg),
10384 Assembler::LSL,
10385 $src3$$constant & 0x3f);
10386 %}
10387
10388 ins_pipe(ialu_reg_reg_shift);
10389 %}
10390
10391 // This pattern is automatically generated from aarch64_ad.m4
10392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10393 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10394 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10395 iRegIorL2I src1, iRegIorL2I src2,
10396 immI src3, immI_M1 src4) %{
10397 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10398 ins_cost(1.9 * INSN_COST);
10399 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10400
10401 ins_encode %{
10402 __ eonw(as_Register($dst$$reg),
10403 as_Register($src1$$reg),
10404 as_Register($src2$$reg),
10405 Assembler::LSR,
10406 $src3$$constant & 0x1f);
10407 %}
10408
10409 ins_pipe(ialu_reg_reg_shift);
10410 %}
10411
10412 // This pattern is automatically generated from aarch64_ad.m4
10413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10414 // val ^ (-1 ^ (val >>> shift)) ==> eon
10415 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10416 iRegL src1, iRegL src2,
10417 immI src3, immL_M1 src4) %{
10418 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10419 ins_cost(1.9 * INSN_COST);
10420 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10421
10422 ins_encode %{
10423 __ eon(as_Register($dst$$reg),
10424 as_Register($src1$$reg),
10425 as_Register($src2$$reg),
10426 Assembler::LSR,
10427 $src3$$constant & 0x3f);
10428 %}
10429
10430 ins_pipe(ialu_reg_reg_shift);
10431 %}
10432
10433 // This pattern is automatically generated from aarch64_ad.m4
10434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10435 // val ^ (-1 ^ (val >> shift)) ==> eonw
10436 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10437 iRegIorL2I src1, iRegIorL2I src2,
10438 immI src3, immI_M1 src4) %{
10439 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10440 ins_cost(1.9 * INSN_COST);
10441 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10442
10443 ins_encode %{
10444 __ eonw(as_Register($dst$$reg),
10445 as_Register($src1$$reg),
10446 as_Register($src2$$reg),
10447 Assembler::ASR,
10448 $src3$$constant & 0x1f);
10449 %}
10450
10451 ins_pipe(ialu_reg_reg_shift);
10452 %}
10453
10454 // This pattern is automatically generated from aarch64_ad.m4
10455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10456 // val ^ (-1 ^ (val >> shift)) ==> eon
10457 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10458 iRegL src1, iRegL src2,
10459 immI src3, immL_M1 src4) %{
10460 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10461 ins_cost(1.9 * INSN_COST);
10462 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10463
10464 ins_encode %{
10465 __ eon(as_Register($dst$$reg),
10466 as_Register($src1$$reg),
10467 as_Register($src2$$reg),
10468 Assembler::ASR,
10469 $src3$$constant & 0x3f);
10470 %}
10471
10472 ins_pipe(ialu_reg_reg_shift);
10473 %}
10474
10475 // This pattern is automatically generated from aarch64_ad.m4
10476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10477 // val ^ (-1 ^ (val ror shift)) ==> eonw
10478 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10479 iRegIorL2I src1, iRegIorL2I src2,
10480 immI src3, immI_M1 src4) %{
10481 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10482 ins_cost(1.9 * INSN_COST);
10483 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10484
10485 ins_encode %{
10486 __ eonw(as_Register($dst$$reg),
10487 as_Register($src1$$reg),
10488 as_Register($src2$$reg),
10489 Assembler::ROR,
10490 $src3$$constant & 0x1f);
10491 %}
10492
10493 ins_pipe(ialu_reg_reg_shift);
10494 %}
10495
10496 // This pattern is automatically generated from aarch64_ad.m4
10497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10498 // val ^ (-1 ^ (val ror shift)) ==> eon
10499 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10500 iRegL src1, iRegL src2,
10501 immI src3, immL_M1 src4) %{
10502 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10503 ins_cost(1.9 * INSN_COST);
10504 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10505
10506 ins_encode %{
10507 __ eon(as_Register($dst$$reg),
10508 as_Register($src1$$reg),
10509 as_Register($src2$$reg),
10510 Assembler::ROR,
10511 $src3$$constant & 0x3f);
10512 %}
10513
10514 ins_pipe(ialu_reg_reg_shift);
10515 %}
10516
10517 // This pattern is automatically generated from aarch64_ad.m4
10518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10519 // val ^ (-1 ^ (val << shift)) ==> eonw
10520 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10521 iRegIorL2I src1, iRegIorL2I src2,
10522 immI src3, immI_M1 src4) %{
10523 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10524 ins_cost(1.9 * INSN_COST);
10525 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10526
10527 ins_encode %{
10528 __ eonw(as_Register($dst$$reg),
10529 as_Register($src1$$reg),
10530 as_Register($src2$$reg),
10531 Assembler::LSL,
10532 $src3$$constant & 0x1f);
10533 %}
10534
10535 ins_pipe(ialu_reg_reg_shift);
10536 %}
10537
10538 // This pattern is automatically generated from aarch64_ad.m4
10539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10540 // val ^ (-1 ^ (val << shift)) ==> eon
10541 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10542 iRegL src1, iRegL src2,
10543 immI src3, immL_M1 src4) %{
10544 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10545 ins_cost(1.9 * INSN_COST);
10546 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10547
10548 ins_encode %{
10549 __ eon(as_Register($dst$$reg),
10550 as_Register($src1$$reg),
10551 as_Register($src2$$reg),
10552 Assembler::LSL,
10553 $src3$$constant & 0x3f);
10554 %}
10555
10556 ins_pipe(ialu_reg_reg_shift);
10557 %}
10558
10559 // This pattern is automatically generated from aarch64_ad.m4
10560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10561 // val | (-1 ^ (val >>> shift)) ==> ornw
10562 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10563 iRegIorL2I src1, iRegIorL2I src2,
10564 immI src3, immI_M1 src4) %{
10565 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10566 ins_cost(1.9 * INSN_COST);
10567 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10568
10569 ins_encode %{
10570 __ ornw(as_Register($dst$$reg),
10571 as_Register($src1$$reg),
10572 as_Register($src2$$reg),
10573 Assembler::LSR,
10574 $src3$$constant & 0x1f);
10575 %}
10576
10577 ins_pipe(ialu_reg_reg_shift);
10578 %}
10579
10580 // This pattern is automatically generated from aarch64_ad.m4
10581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10582 // val | (-1 ^ (val >>> shift)) ==> orn
10583 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10584 iRegL src1, iRegL src2,
10585 immI src3, immL_M1 src4) %{
10586 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10587 ins_cost(1.9 * INSN_COST);
10588 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10589
10590 ins_encode %{
10591 __ orn(as_Register($dst$$reg),
10592 as_Register($src1$$reg),
10593 as_Register($src2$$reg),
10594 Assembler::LSR,
10595 $src3$$constant & 0x3f);
10596 %}
10597
10598 ins_pipe(ialu_reg_reg_shift);
10599 %}
10600
10601 // This pattern is automatically generated from aarch64_ad.m4
10602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10603 // val | (-1 ^ (val >> shift)) ==> ornw
10604 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10605 iRegIorL2I src1, iRegIorL2I src2,
10606 immI src3, immI_M1 src4) %{
10607 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10608 ins_cost(1.9 * INSN_COST);
10609 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10610
10611 ins_encode %{
10612 __ ornw(as_Register($dst$$reg),
10613 as_Register($src1$$reg),
10614 as_Register($src2$$reg),
10615 Assembler::ASR,
10616 $src3$$constant & 0x1f);
10617 %}
10618
10619 ins_pipe(ialu_reg_reg_shift);
10620 %}
10621
10622 // This pattern is automatically generated from aarch64_ad.m4
10623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10624 // val | (-1 ^ (val >> shift)) ==> orn
10625 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10626 iRegL src1, iRegL src2,
10627 immI src3, immL_M1 src4) %{
10628 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10629 ins_cost(1.9 * INSN_COST);
10630 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10631
10632 ins_encode %{
10633 __ orn(as_Register($dst$$reg),
10634 as_Register($src1$$reg),
10635 as_Register($src2$$reg),
10636 Assembler::ASR,
10637 $src3$$constant & 0x3f);
10638 %}
10639
10640 ins_pipe(ialu_reg_reg_shift);
10641 %}
10642
10643 // This pattern is automatically generated from aarch64_ad.m4
10644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10645 // val | (-1 ^ (val ror shift)) ==> ornw
10646 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10647 iRegIorL2I src1, iRegIorL2I src2,
10648 immI src3, immI_M1 src4) %{
10649 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10650 ins_cost(1.9 * INSN_COST);
10651 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10652
10653 ins_encode %{
10654 __ ornw(as_Register($dst$$reg),
10655 as_Register($src1$$reg),
10656 as_Register($src2$$reg),
10657 Assembler::ROR,
10658 $src3$$constant & 0x1f);
10659 %}
10660
10661 ins_pipe(ialu_reg_reg_shift);
10662 %}
10663
10664 // This pattern is automatically generated from aarch64_ad.m4
10665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10666 // val | (-1 ^ (val ror shift)) ==> orn
10667 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10668 iRegL src1, iRegL src2,
10669 immI src3, immL_M1 src4) %{
10670 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10671 ins_cost(1.9 * INSN_COST);
10672 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10673
10674 ins_encode %{
10675 __ orn(as_Register($dst$$reg),
10676 as_Register($src1$$reg),
10677 as_Register($src2$$reg),
10678 Assembler::ROR,
10679 $src3$$constant & 0x3f);
10680 %}
10681
10682 ins_pipe(ialu_reg_reg_shift);
10683 %}
10684
10685 // This pattern is automatically generated from aarch64_ad.m4
10686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10687 // val | (-1 ^ (val << shift)) ==> ornw
10688 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10689 iRegIorL2I src1, iRegIorL2I src2,
10690 immI src3, immI_M1 src4) %{
10691 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10692 ins_cost(1.9 * INSN_COST);
10693 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10694
10695 ins_encode %{
10696 __ ornw(as_Register($dst$$reg),
10697 as_Register($src1$$reg),
10698 as_Register($src2$$reg),
10699 Assembler::LSL,
10700 $src3$$constant & 0x1f);
10701 %}
10702
10703 ins_pipe(ialu_reg_reg_shift);
10704 %}
10705
10706 // This pattern is automatically generated from aarch64_ad.m4
10707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10708 // val | (-1 ^ (val << shift)) ==> orn
10709 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10710 iRegL src1, iRegL src2,
10711 immI src3, immL_M1 src4) %{
10712 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10713 ins_cost(1.9 * INSN_COST);
10714 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10715
10716 ins_encode %{
10717 __ orn(as_Register($dst$$reg),
10718 as_Register($src1$$reg),
10719 as_Register($src2$$reg),
10720 Assembler::LSL,
10721 $src3$$constant & 0x3f);
10722 %}
10723
10724 ins_pipe(ialu_reg_reg_shift);
10725 %}
10726
10727 // This pattern is automatically generated from aarch64_ad.m4
10728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10729 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10730 iRegIorL2I src1, iRegIorL2I src2,
10731 immI src3) %{
10732 match(Set dst (AndI src1 (URShiftI src2 src3)));
10733
10734 ins_cost(1.9 * INSN_COST);
10735 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10736
10737 ins_encode %{
10738 __ andw(as_Register($dst$$reg),
10739 as_Register($src1$$reg),
10740 as_Register($src2$$reg),
10741 Assembler::LSR,
10742 $src3$$constant & 0x1f);
10743 %}
10744
10745 ins_pipe(ialu_reg_reg_shift);
10746 %}
10747
10748 // This pattern is automatically generated from aarch64_ad.m4
10749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10750 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10751 iRegL src1, iRegL src2,
10752 immI src3) %{
10753 match(Set dst (AndL src1 (URShiftL src2 src3)));
10754
10755 ins_cost(1.9 * INSN_COST);
10756 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10757
10758 ins_encode %{
10759 __ andr(as_Register($dst$$reg),
10760 as_Register($src1$$reg),
10761 as_Register($src2$$reg),
10762 Assembler::LSR,
10763 $src3$$constant & 0x3f);
10764 %}
10765
10766 ins_pipe(ialu_reg_reg_shift);
10767 %}
10768
10769 // This pattern is automatically generated from aarch64_ad.m4
10770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10771 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10772 iRegIorL2I src1, iRegIorL2I src2,
10773 immI src3) %{
10774 match(Set dst (AndI src1 (RShiftI src2 src3)));
10775
10776 ins_cost(1.9 * INSN_COST);
10777 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10778
10779 ins_encode %{
10780 __ andw(as_Register($dst$$reg),
10781 as_Register($src1$$reg),
10782 as_Register($src2$$reg),
10783 Assembler::ASR,
10784 $src3$$constant & 0x1f);
10785 %}
10786
10787 ins_pipe(ialu_reg_reg_shift);
10788 %}
10789
10790 // This pattern is automatically generated from aarch64_ad.m4
10791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10792 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10793 iRegL src1, iRegL src2,
10794 immI src3) %{
10795 match(Set dst (AndL src1 (RShiftL src2 src3)));
10796
10797 ins_cost(1.9 * INSN_COST);
10798 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10799
10800 ins_encode %{
10801 __ andr(as_Register($dst$$reg),
10802 as_Register($src1$$reg),
10803 as_Register($src2$$reg),
10804 Assembler::ASR,
10805 $src3$$constant & 0x3f);
10806 %}
10807
10808 ins_pipe(ialu_reg_reg_shift);
10809 %}
10810
10811 // This pattern is automatically generated from aarch64_ad.m4
10812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10813 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10814 iRegIorL2I src1, iRegIorL2I src2,
10815 immI src3) %{
10816 match(Set dst (AndI src1 (LShiftI src2 src3)));
10817
10818 ins_cost(1.9 * INSN_COST);
10819 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10820
10821 ins_encode %{
10822 __ andw(as_Register($dst$$reg),
10823 as_Register($src1$$reg),
10824 as_Register($src2$$reg),
10825 Assembler::LSL,
10826 $src3$$constant & 0x1f);
10827 %}
10828
10829 ins_pipe(ialu_reg_reg_shift);
10830 %}
10831
10832 // This pattern is automatically generated from aarch64_ad.m4
10833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10834 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10835 iRegL src1, iRegL src2,
10836 immI src3) %{
10837 match(Set dst (AndL src1 (LShiftL src2 src3)));
10838
10839 ins_cost(1.9 * INSN_COST);
10840 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10841
10842 ins_encode %{
10843 __ andr(as_Register($dst$$reg),
10844 as_Register($src1$$reg),
10845 as_Register($src2$$reg),
10846 Assembler::LSL,
10847 $src3$$constant & 0x3f);
10848 %}
10849
10850 ins_pipe(ialu_reg_reg_shift);
10851 %}
10852
10853 // This pattern is automatically generated from aarch64_ad.m4
10854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10855 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10856 iRegIorL2I src1, iRegIorL2I src2,
10857 immI src3) %{
10858 match(Set dst (AndI src1 (RotateRight src2 src3)));
10859
10860 ins_cost(1.9 * INSN_COST);
10861 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10862
10863 ins_encode %{
10864 __ andw(as_Register($dst$$reg),
10865 as_Register($src1$$reg),
10866 as_Register($src2$$reg),
10867 Assembler::ROR,
10868 $src3$$constant & 0x1f);
10869 %}
10870
10871 ins_pipe(ialu_reg_reg_shift);
10872 %}
10873
10874 // This pattern is automatically generated from aarch64_ad.m4
10875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10876 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10877 iRegL src1, iRegL src2,
10878 immI src3) %{
10879 match(Set dst (AndL src1 (RotateRight src2 src3)));
10880
10881 ins_cost(1.9 * INSN_COST);
10882 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10883
10884 ins_encode %{
10885 __ andr(as_Register($dst$$reg),
10886 as_Register($src1$$reg),
10887 as_Register($src2$$reg),
10888 Assembler::ROR,
10889 $src3$$constant & 0x3f);
10890 %}
10891
10892 ins_pipe(ialu_reg_reg_shift);
10893 %}
10894
10895 // This pattern is automatically generated from aarch64_ad.m4
10896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10897 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10898 iRegIorL2I src1, iRegIorL2I src2,
10899 immI src3) %{
10900 match(Set dst (XorI src1 (URShiftI src2 src3)));
10901
10902 ins_cost(1.9 * INSN_COST);
10903 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10904
10905 ins_encode %{
10906 __ eorw(as_Register($dst$$reg),
10907 as_Register($src1$$reg),
10908 as_Register($src2$$reg),
10909 Assembler::LSR,
10910 $src3$$constant & 0x1f);
10911 %}
10912
10913 ins_pipe(ialu_reg_reg_shift);
10914 %}
10915
10916 // This pattern is automatically generated from aarch64_ad.m4
10917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10918 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10919 iRegL src1, iRegL src2,
10920 immI src3) %{
10921 match(Set dst (XorL src1 (URShiftL src2 src3)));
10922
10923 ins_cost(1.9 * INSN_COST);
10924 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10925
10926 ins_encode %{
10927 __ eor(as_Register($dst$$reg),
10928 as_Register($src1$$reg),
10929 as_Register($src2$$reg),
10930 Assembler::LSR,
10931 $src3$$constant & 0x3f);
10932 %}
10933
10934 ins_pipe(ialu_reg_reg_shift);
10935 %}
10936
10937 // This pattern is automatically generated from aarch64_ad.m4
10938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10939 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10940 iRegIorL2I src1, iRegIorL2I src2,
10941 immI src3) %{
10942 match(Set dst (XorI src1 (RShiftI src2 src3)));
10943
10944 ins_cost(1.9 * INSN_COST);
10945 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10946
10947 ins_encode %{
10948 __ eorw(as_Register($dst$$reg),
10949 as_Register($src1$$reg),
10950 as_Register($src2$$reg),
10951 Assembler::ASR,
10952 $src3$$constant & 0x1f);
10953 %}
10954
10955 ins_pipe(ialu_reg_reg_shift);
10956 %}
10957
10958 // This pattern is automatically generated from aarch64_ad.m4
10959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10960 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10961 iRegL src1, iRegL src2,
10962 immI src3) %{
10963 match(Set dst (XorL src1 (RShiftL src2 src3)));
10964
10965 ins_cost(1.9 * INSN_COST);
10966 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10967
10968 ins_encode %{
10969 __ eor(as_Register($dst$$reg),
10970 as_Register($src1$$reg),
10971 as_Register($src2$$reg),
10972 Assembler::ASR,
10973 $src3$$constant & 0x3f);
10974 %}
10975
10976 ins_pipe(ialu_reg_reg_shift);
10977 %}
10978
10979 // This pattern is automatically generated from aarch64_ad.m4
10980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10981 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10982 iRegIorL2I src1, iRegIorL2I src2,
10983 immI src3) %{
10984 match(Set dst (XorI src1 (LShiftI src2 src3)));
10985
10986 ins_cost(1.9 * INSN_COST);
10987 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10988
10989 ins_encode %{
10990 __ eorw(as_Register($dst$$reg),
10991 as_Register($src1$$reg),
10992 as_Register($src2$$reg),
10993 Assembler::LSL,
10994 $src3$$constant & 0x1f);
10995 %}
10996
10997 ins_pipe(ialu_reg_reg_shift);
10998 %}
10999
11000 // This pattern is automatically generated from aarch64_ad.m4
11001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11002 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11003 iRegL src1, iRegL src2,
11004 immI src3) %{
11005 match(Set dst (XorL src1 (LShiftL src2 src3)));
11006
11007 ins_cost(1.9 * INSN_COST);
11008 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11009
11010 ins_encode %{
11011 __ eor(as_Register($dst$$reg),
11012 as_Register($src1$$reg),
11013 as_Register($src2$$reg),
11014 Assembler::LSL,
11015 $src3$$constant & 0x3f);
11016 %}
11017
11018 ins_pipe(ialu_reg_reg_shift);
11019 %}
11020
11021 // This pattern is automatically generated from aarch64_ad.m4
11022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11023 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11024 iRegIorL2I src1, iRegIorL2I src2,
11025 immI src3) %{
11026 match(Set dst (XorI src1 (RotateRight src2 src3)));
11027
11028 ins_cost(1.9 * INSN_COST);
11029 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11030
11031 ins_encode %{
11032 __ eorw(as_Register($dst$$reg),
11033 as_Register($src1$$reg),
11034 as_Register($src2$$reg),
11035 Assembler::ROR,
11036 $src3$$constant & 0x1f);
11037 %}
11038
11039 ins_pipe(ialu_reg_reg_shift);
11040 %}
11041
11042 // This pattern is automatically generated from aarch64_ad.m4
11043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11044 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11045 iRegL src1, iRegL src2,
11046 immI src3) %{
11047 match(Set dst (XorL src1 (RotateRight src2 src3)));
11048
11049 ins_cost(1.9 * INSN_COST);
11050 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11051
11052 ins_encode %{
11053 __ eor(as_Register($dst$$reg),
11054 as_Register($src1$$reg),
11055 as_Register($src2$$reg),
11056 Assembler::ROR,
11057 $src3$$constant & 0x3f);
11058 %}
11059
11060 ins_pipe(ialu_reg_reg_shift);
11061 %}
11062
11063 // This pattern is automatically generated from aarch64_ad.m4
11064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11065 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11066 iRegIorL2I src1, iRegIorL2I src2,
11067 immI src3) %{
11068 match(Set dst (OrI src1 (URShiftI src2 src3)));
11069
11070 ins_cost(1.9 * INSN_COST);
11071 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11072
11073 ins_encode %{
11074 __ orrw(as_Register($dst$$reg),
11075 as_Register($src1$$reg),
11076 as_Register($src2$$reg),
11077 Assembler::LSR,
11078 $src3$$constant & 0x1f);
11079 %}
11080
11081 ins_pipe(ialu_reg_reg_shift);
11082 %}
11083
11084 // This pattern is automatically generated from aarch64_ad.m4
11085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11086 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11087 iRegL src1, iRegL src2,
11088 immI src3) %{
11089 match(Set dst (OrL src1 (URShiftL src2 src3)));
11090
11091 ins_cost(1.9 * INSN_COST);
11092 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11093
11094 ins_encode %{
11095 __ orr(as_Register($dst$$reg),
11096 as_Register($src1$$reg),
11097 as_Register($src2$$reg),
11098 Assembler::LSR,
11099 $src3$$constant & 0x3f);
11100 %}
11101
11102 ins_pipe(ialu_reg_reg_shift);
11103 %}
11104
11105 // This pattern is automatically generated from aarch64_ad.m4
11106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11107 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11108 iRegIorL2I src1, iRegIorL2I src2,
11109 immI src3) %{
11110 match(Set dst (OrI src1 (RShiftI src2 src3)));
11111
11112 ins_cost(1.9 * INSN_COST);
11113 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11114
11115 ins_encode %{
11116 __ orrw(as_Register($dst$$reg),
11117 as_Register($src1$$reg),
11118 as_Register($src2$$reg),
11119 Assembler::ASR,
11120 $src3$$constant & 0x1f);
11121 %}
11122
11123 ins_pipe(ialu_reg_reg_shift);
11124 %}
11125
11126 // This pattern is automatically generated from aarch64_ad.m4
11127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11128 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11129 iRegL src1, iRegL src2,
11130 immI src3) %{
11131 match(Set dst (OrL src1 (RShiftL src2 src3)));
11132
11133 ins_cost(1.9 * INSN_COST);
11134 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11135
11136 ins_encode %{
11137 __ orr(as_Register($dst$$reg),
11138 as_Register($src1$$reg),
11139 as_Register($src2$$reg),
11140 Assembler::ASR,
11141 $src3$$constant & 0x3f);
11142 %}
11143
11144 ins_pipe(ialu_reg_reg_shift);
11145 %}
11146
11147 // This pattern is automatically generated from aarch64_ad.m4
11148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11149 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11150 iRegIorL2I src1, iRegIorL2I src2,
11151 immI src3) %{
11152 match(Set dst (OrI src1 (LShiftI src2 src3)));
11153
11154 ins_cost(1.9 * INSN_COST);
11155 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11156
11157 ins_encode %{
11158 __ orrw(as_Register($dst$$reg),
11159 as_Register($src1$$reg),
11160 as_Register($src2$$reg),
11161 Assembler::LSL,
11162 $src3$$constant & 0x1f);
11163 %}
11164
11165 ins_pipe(ialu_reg_reg_shift);
11166 %}
11167
11168 // This pattern is automatically generated from aarch64_ad.m4
11169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11170 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11171 iRegL src1, iRegL src2,
11172 immI src3) %{
11173 match(Set dst (OrL src1 (LShiftL src2 src3)));
11174
11175 ins_cost(1.9 * INSN_COST);
11176 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11177
11178 ins_encode %{
11179 __ orr(as_Register($dst$$reg),
11180 as_Register($src1$$reg),
11181 as_Register($src2$$reg),
11182 Assembler::LSL,
11183 $src3$$constant & 0x3f);
11184 %}
11185
11186 ins_pipe(ialu_reg_reg_shift);
11187 %}
11188
11189 // This pattern is automatically generated from aarch64_ad.m4
11190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11191 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11192 iRegIorL2I src1, iRegIorL2I src2,
11193 immI src3) %{
11194 match(Set dst (OrI src1 (RotateRight src2 src3)));
11195
11196 ins_cost(1.9 * INSN_COST);
11197 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11198
11199 ins_encode %{
11200 __ orrw(as_Register($dst$$reg),
11201 as_Register($src1$$reg),
11202 as_Register($src2$$reg),
11203 Assembler::ROR,
11204 $src3$$constant & 0x1f);
11205 %}
11206
11207 ins_pipe(ialu_reg_reg_shift);
11208 %}
11209
11210 // This pattern is automatically generated from aarch64_ad.m4
11211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11212 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11213 iRegL src1, iRegL src2,
11214 immI src3) %{
11215 match(Set dst (OrL src1 (RotateRight src2 src3)));
11216
11217 ins_cost(1.9 * INSN_COST);
11218 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11219
11220 ins_encode %{
11221 __ orr(as_Register($dst$$reg),
11222 as_Register($src1$$reg),
11223 as_Register($src2$$reg),
11224 Assembler::ROR,
11225 $src3$$constant & 0x3f);
11226 %}
11227
11228 ins_pipe(ialu_reg_reg_shift);
11229 %}
11230
11231 // This pattern is automatically generated from aarch64_ad.m4
11232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11233 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11234 iRegIorL2I src1, iRegIorL2I src2,
11235 immI src3) %{
11236 match(Set dst (AddI src1 (URShiftI src2 src3)));
11237
11238 ins_cost(1.9 * INSN_COST);
11239 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11240
11241 ins_encode %{
11242 __ addw(as_Register($dst$$reg),
11243 as_Register($src1$$reg),
11244 as_Register($src2$$reg),
11245 Assembler::LSR,
11246 $src3$$constant & 0x1f);
11247 %}
11248
11249 ins_pipe(ialu_reg_reg_shift);
11250 %}
11251
11252 // This pattern is automatically generated from aarch64_ad.m4
11253 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11254 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11255 iRegL src1, iRegL src2,
11256 immI src3) %{
11257 match(Set dst (AddL src1 (URShiftL src2 src3)));
11258
11259 ins_cost(1.9 * INSN_COST);
11260 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11261
11262 ins_encode %{
11263 __ add(as_Register($dst$$reg),
11264 as_Register($src1$$reg),
11265 as_Register($src2$$reg),
11266 Assembler::LSR,
11267 $src3$$constant & 0x3f);
11268 %}
11269
11270 ins_pipe(ialu_reg_reg_shift);
11271 %}
11272
11273 // This pattern is automatically generated from aarch64_ad.m4
11274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11275 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11276 iRegIorL2I src1, iRegIorL2I src2,
11277 immI src3) %{
11278 match(Set dst (AddI src1 (RShiftI src2 src3)));
11279
11280 ins_cost(1.9 * INSN_COST);
11281 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11282
11283 ins_encode %{
11284 __ addw(as_Register($dst$$reg),
11285 as_Register($src1$$reg),
11286 as_Register($src2$$reg),
11287 Assembler::ASR,
11288 $src3$$constant & 0x1f);
11289 %}
11290
11291 ins_pipe(ialu_reg_reg_shift);
11292 %}
11293
11294 // This pattern is automatically generated from aarch64_ad.m4
11295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11296 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11297 iRegL src1, iRegL src2,
11298 immI src3) %{
11299 match(Set dst (AddL src1 (RShiftL src2 src3)));
11300
11301 ins_cost(1.9 * INSN_COST);
11302 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11303
11304 ins_encode %{
11305 __ add(as_Register($dst$$reg),
11306 as_Register($src1$$reg),
11307 as_Register($src2$$reg),
11308 Assembler::ASR,
11309 $src3$$constant & 0x3f);
11310 %}
11311
11312 ins_pipe(ialu_reg_reg_shift);
11313 %}
11314
11315 // This pattern is automatically generated from aarch64_ad.m4
11316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11317 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11318 iRegIorL2I src1, iRegIorL2I src2,
11319 immI src3) %{
11320 match(Set dst (AddI src1 (LShiftI src2 src3)));
11321
11322 ins_cost(1.9 * INSN_COST);
11323 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11324
11325 ins_encode %{
11326 __ addw(as_Register($dst$$reg),
11327 as_Register($src1$$reg),
11328 as_Register($src2$$reg),
11329 Assembler::LSL,
11330 $src3$$constant & 0x1f);
11331 %}
11332
11333 ins_pipe(ialu_reg_reg_shift);
11334 %}
11335
11336 // This pattern is automatically generated from aarch64_ad.m4
11337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11338 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11339 iRegL src1, iRegL src2,
11340 immI src3) %{
11341 match(Set dst (AddL src1 (LShiftL src2 src3)));
11342
11343 ins_cost(1.9 * INSN_COST);
11344 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11345
11346 ins_encode %{
11347 __ add(as_Register($dst$$reg),
11348 as_Register($src1$$reg),
11349 as_Register($src2$$reg),
11350 Assembler::LSL,
11351 $src3$$constant & 0x3f);
11352 %}
11353
11354 ins_pipe(ialu_reg_reg_shift);
11355 %}
11356
11357 // This pattern is automatically generated from aarch64_ad.m4
11358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11359 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11360 iRegIorL2I src1, iRegIorL2I src2,
11361 immI src3) %{
11362 match(Set dst (SubI src1 (URShiftI src2 src3)));
11363
11364 ins_cost(1.9 * INSN_COST);
11365 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11366
11367 ins_encode %{
11368 __ subw(as_Register($dst$$reg),
11369 as_Register($src1$$reg),
11370 as_Register($src2$$reg),
11371 Assembler::LSR,
11372 $src3$$constant & 0x1f);
11373 %}
11374
11375 ins_pipe(ialu_reg_reg_shift);
11376 %}
11377
11378 // This pattern is automatically generated from aarch64_ad.m4
11379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11380 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11381 iRegL src1, iRegL src2,
11382 immI src3) %{
11383 match(Set dst (SubL src1 (URShiftL src2 src3)));
11384
11385 ins_cost(1.9 * INSN_COST);
11386 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11387
11388 ins_encode %{
11389 __ sub(as_Register($dst$$reg),
11390 as_Register($src1$$reg),
11391 as_Register($src2$$reg),
11392 Assembler::LSR,
11393 $src3$$constant & 0x3f);
11394 %}
11395
11396 ins_pipe(ialu_reg_reg_shift);
11397 %}
11398
11399 // This pattern is automatically generated from aarch64_ad.m4
11400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11401 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11402 iRegIorL2I src1, iRegIorL2I src2,
11403 immI src3) %{
11404 match(Set dst (SubI src1 (RShiftI src2 src3)));
11405
11406 ins_cost(1.9 * INSN_COST);
11407 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11408
11409 ins_encode %{
11410 __ subw(as_Register($dst$$reg),
11411 as_Register($src1$$reg),
11412 as_Register($src2$$reg),
11413 Assembler::ASR,
11414 $src3$$constant & 0x1f);
11415 %}
11416
11417 ins_pipe(ialu_reg_reg_shift);
11418 %}
11419
11420 // This pattern is automatically generated from aarch64_ad.m4
11421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11422 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11423 iRegL src1, iRegL src2,
11424 immI src3) %{
11425 match(Set dst (SubL src1 (RShiftL src2 src3)));
11426
11427 ins_cost(1.9 * INSN_COST);
11428 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11429
11430 ins_encode %{
11431 __ sub(as_Register($dst$$reg),
11432 as_Register($src1$$reg),
11433 as_Register($src2$$reg),
11434 Assembler::ASR,
11435 $src3$$constant & 0x3f);
11436 %}
11437
11438 ins_pipe(ialu_reg_reg_shift);
11439 %}
11440
11441 // This pattern is automatically generated from aarch64_ad.m4
11442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11443 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11444 iRegIorL2I src1, iRegIorL2I src2,
11445 immI src3) %{
11446 match(Set dst (SubI src1 (LShiftI src2 src3)));
11447
11448 ins_cost(1.9 * INSN_COST);
11449 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11450
11451 ins_encode %{
11452 __ subw(as_Register($dst$$reg),
11453 as_Register($src1$$reg),
11454 as_Register($src2$$reg),
11455 Assembler::LSL,
11456 $src3$$constant & 0x1f);
11457 %}
11458
11459 ins_pipe(ialu_reg_reg_shift);
11460 %}
11461
11462 // This pattern is automatically generated from aarch64_ad.m4
11463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11464 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11465 iRegL src1, iRegL src2,
11466 immI src3) %{
11467 match(Set dst (SubL src1 (LShiftL src2 src3)));
11468
11469 ins_cost(1.9 * INSN_COST);
11470 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11471
11472 ins_encode %{
11473 __ sub(as_Register($dst$$reg),
11474 as_Register($src1$$reg),
11475 as_Register($src2$$reg),
11476 Assembler::LSL,
11477 $src3$$constant & 0x3f);
11478 %}
11479
11480 ins_pipe(ialu_reg_reg_shift);
11481 %}
11482
11483 // This pattern is automatically generated from aarch64_ad.m4
11484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11485
11486 // Shift Left followed by Shift Right.
11487 // This idiom is used by the compiler for the i2b bytecode etc.
11488 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11489 %{
11490 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11491 ins_cost(INSN_COST * 2);
11492 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11493 ins_encode %{
11494 int lshift = $lshift_count$$constant & 63;
11495 int rshift = $rshift_count$$constant & 63;
11496 int s = 63 - lshift;
11497 int r = (rshift - lshift) & 63;
11498 __ sbfm(as_Register($dst$$reg),
11499 as_Register($src$$reg),
11500 r, s);
11501 %}
11502
11503 ins_pipe(ialu_reg_shift);
11504 %}
11505
11506 // This pattern is automatically generated from aarch64_ad.m4
11507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11508
11509 // Shift Left followed by Shift Right.
11510 // This idiom is used by the compiler for the i2b bytecode etc.
11511 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11512 %{
11513 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11514 ins_cost(INSN_COST * 2);
11515 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11516 ins_encode %{
11517 int lshift = $lshift_count$$constant & 31;
11518 int rshift = $rshift_count$$constant & 31;
11519 int s = 31 - lshift;
11520 int r = (rshift - lshift) & 31;
11521 __ sbfmw(as_Register($dst$$reg),
11522 as_Register($src$$reg),
11523 r, s);
11524 %}
11525
11526 ins_pipe(ialu_reg_shift);
11527 %}
11528
11529 // This pattern is automatically generated from aarch64_ad.m4
11530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11531
11532 // Shift Left followed by Shift Right.
11533 // This idiom is used by the compiler for the i2b bytecode etc.
11534 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11535 %{
11536 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11537 ins_cost(INSN_COST * 2);
11538 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11539 ins_encode %{
11540 int lshift = $lshift_count$$constant & 63;
11541 int rshift = $rshift_count$$constant & 63;
11542 int s = 63 - lshift;
11543 int r = (rshift - lshift) & 63;
11544 __ ubfm(as_Register($dst$$reg),
11545 as_Register($src$$reg),
11546 r, s);
11547 %}
11548
11549 ins_pipe(ialu_reg_shift);
11550 %}
11551
11552 // This pattern is automatically generated from aarch64_ad.m4
11553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11554
11555 // Shift Left followed by Shift Right.
11556 // This idiom is used by the compiler for the i2b bytecode etc.
11557 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11558 %{
11559 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11560 ins_cost(INSN_COST * 2);
11561 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11562 ins_encode %{
11563 int lshift = $lshift_count$$constant & 31;
11564 int rshift = $rshift_count$$constant & 31;
11565 int s = 31 - lshift;
11566 int r = (rshift - lshift) & 31;
11567 __ ubfmw(as_Register($dst$$reg),
11568 as_Register($src$$reg),
11569 r, s);
11570 %}
11571
11572 ins_pipe(ialu_reg_shift);
11573 %}
11574
11575 // Bitfield extract with shift & mask
11576
11577 // This pattern is automatically generated from aarch64_ad.m4
11578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11579 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11580 %{
11581 match(Set dst (AndI (URShiftI src rshift) mask));
11582 // Make sure we are not going to exceed what ubfxw can do.
11583 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11584
11585 ins_cost(INSN_COST);
11586 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11587 ins_encode %{
11588 int rshift = $rshift$$constant & 31;
11589 intptr_t mask = $mask$$constant;
11590 int width = exact_log2(mask+1);
11591 __ ubfxw(as_Register($dst$$reg),
11592 as_Register($src$$reg), rshift, width);
11593 %}
11594 ins_pipe(ialu_reg_shift);
11595 %}
11596
11597 // This pattern is automatically generated from aarch64_ad.m4
11598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11599 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11600 %{
11601 match(Set dst (AndL (URShiftL src rshift) mask));
11602 // Make sure we are not going to exceed what ubfx can do.
11603 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11604
11605 ins_cost(INSN_COST);
11606 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11607 ins_encode %{
11608 int rshift = $rshift$$constant & 63;
11609 intptr_t mask = $mask$$constant;
11610 int width = exact_log2_long(mask+1);
11611 __ ubfx(as_Register($dst$$reg),
11612 as_Register($src$$reg), rshift, width);
11613 %}
11614 ins_pipe(ialu_reg_shift);
11615 %}
11616
11617
11618 // This pattern is automatically generated from aarch64_ad.m4
11619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11620
11621 // We can use ubfx when extending an And with a mask when we know mask
11622 // is positive. We know that because immI_bitmask guarantees it.
11623 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11624 %{
11625 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11626 // Make sure we are not going to exceed what ubfxw can do.
11627 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11628
11629 ins_cost(INSN_COST * 2);
11630 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11631 ins_encode %{
11632 int rshift = $rshift$$constant & 31;
11633 intptr_t mask = $mask$$constant;
11634 int width = exact_log2(mask+1);
11635 __ ubfx(as_Register($dst$$reg),
11636 as_Register($src$$reg), rshift, width);
11637 %}
11638 ins_pipe(ialu_reg_shift);
11639 %}
11640
11641
11642 // This pattern is automatically generated from aarch64_ad.m4
11643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11644
11645 // We can use ubfiz when masking by a positive number and then left shifting the result.
11646 // We know that the mask is positive because immI_bitmask guarantees it.
11647 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11648 %{
11649 match(Set dst (LShiftI (AndI src mask) lshift));
11650 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11651
11652 ins_cost(INSN_COST);
11653 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11654 ins_encode %{
11655 int lshift = $lshift$$constant & 31;
11656 intptr_t mask = $mask$$constant;
11657 int width = exact_log2(mask+1);
11658 __ ubfizw(as_Register($dst$$reg),
11659 as_Register($src$$reg), lshift, width);
11660 %}
11661 ins_pipe(ialu_reg_shift);
11662 %}
11663
11664 // This pattern is automatically generated from aarch64_ad.m4
11665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11666
11667 // We can use ubfiz when masking by a positive number and then left shifting the result.
11668 // We know that the mask is positive because immL_bitmask guarantees it.
11669 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11670 %{
11671 match(Set dst (LShiftL (AndL src mask) lshift));
11672 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11673
11674 ins_cost(INSN_COST);
11675 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11676 ins_encode %{
11677 int lshift = $lshift$$constant & 63;
11678 intptr_t mask = $mask$$constant;
11679 int width = exact_log2_long(mask+1);
11680 __ ubfiz(as_Register($dst$$reg),
11681 as_Register($src$$reg), lshift, width);
11682 %}
11683 ins_pipe(ialu_reg_shift);
11684 %}
11685
11686 // This pattern is automatically generated from aarch64_ad.m4
11687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11688
11689 // We can use ubfiz when masking by a positive number and then left shifting the result.
11690 // We know that the mask is positive because immI_bitmask guarantees it.
11691 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11692 %{
11693 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11694 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11695
11696 ins_cost(INSN_COST);
11697 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11698 ins_encode %{
11699 int lshift = $lshift$$constant & 31;
11700 intptr_t mask = $mask$$constant;
11701 int width = exact_log2(mask+1);
11702 __ ubfizw(as_Register($dst$$reg),
11703 as_Register($src$$reg), lshift, width);
11704 %}
11705 ins_pipe(ialu_reg_shift);
11706 %}
11707
11708 // This pattern is automatically generated from aarch64_ad.m4
11709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11710
11711 // We can use ubfiz when masking by a positive number and then left shifting the result.
11712 // We know that the mask is positive because immL_bitmask guarantees it.
11713 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11714 %{
11715 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11716 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11717
11718 ins_cost(INSN_COST);
11719 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11720 ins_encode %{
11721 int lshift = $lshift$$constant & 63;
11722 intptr_t mask = $mask$$constant;
11723 int width = exact_log2_long(mask+1);
11724 __ ubfiz(as_Register($dst$$reg),
11725 as_Register($src$$reg), lshift, width);
11726 %}
11727 ins_pipe(ialu_reg_shift);
11728 %}
11729
11730
11731 // This pattern is automatically generated from aarch64_ad.m4
11732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11733
11734 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11735 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11736 %{
11737 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11738 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11739
11740 ins_cost(INSN_COST);
11741 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11742 ins_encode %{
11743 int lshift = $lshift$$constant & 63;
11744 intptr_t mask = $mask$$constant;
11745 int width = exact_log2(mask+1);
11746 __ ubfiz(as_Register($dst$$reg),
11747 as_Register($src$$reg), lshift, width);
11748 %}
11749 ins_pipe(ialu_reg_shift);
11750 %}
11751
11752 // This pattern is automatically generated from aarch64_ad.m4
11753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11754
11755 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11756 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11757 %{
11758 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11759 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11760
11761 ins_cost(INSN_COST);
11762 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11763 ins_encode %{
11764 int lshift = $lshift$$constant & 31;
11765 intptr_t mask = $mask$$constant;
11766 int width = exact_log2(mask+1);
11767 __ ubfiz(as_Register($dst$$reg),
11768 as_Register($src$$reg), lshift, width);
11769 %}
11770 ins_pipe(ialu_reg_shift);
11771 %}
11772
11773 // This pattern is automatically generated from aarch64_ad.m4
11774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11775
11776 // Can skip int2long conversions after AND with small bitmask
11777 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11778 %{
11779 match(Set dst (ConvI2L (AndI src msk)));
11780 ins_cost(INSN_COST);
11781 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11782 ins_encode %{
11783 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11784 %}
11785 ins_pipe(ialu_reg_shift);
11786 %}
11787
11788
11789 // Rotations
11790
11791 // This pattern is automatically generated from aarch64_ad.m4
11792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11793 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11794 %{
11795 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11796 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11797
11798 ins_cost(INSN_COST);
11799 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11800
11801 ins_encode %{
11802 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11803 $rshift$$constant & 63);
11804 %}
11805 ins_pipe(ialu_reg_reg_extr);
11806 %}
11807
11808
11809 // This pattern is automatically generated from aarch64_ad.m4
11810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11811 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11812 %{
11813 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11814 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11815
11816 ins_cost(INSN_COST);
11817 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11818
11819 ins_encode %{
11820 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11821 $rshift$$constant & 31);
11822 %}
11823 ins_pipe(ialu_reg_reg_extr);
11824 %}
11825
11826
11827 // This pattern is automatically generated from aarch64_ad.m4
11828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11829 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11830 %{
11831 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11832 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11833
11834 ins_cost(INSN_COST);
11835 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11836
11837 ins_encode %{
11838 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11839 $rshift$$constant & 63);
11840 %}
11841 ins_pipe(ialu_reg_reg_extr);
11842 %}
11843
11844
11845 // This pattern is automatically generated from aarch64_ad.m4
11846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11847 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11848 %{
11849 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11850 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11851
11852 ins_cost(INSN_COST);
11853 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11854
11855 ins_encode %{
11856 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11857 $rshift$$constant & 31);
11858 %}
11859 ins_pipe(ialu_reg_reg_extr);
11860 %}
11861
11862 // This pattern is automatically generated from aarch64_ad.m4
11863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11864 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11865 %{
11866 match(Set dst (RotateRight src shift));
11867
11868 ins_cost(INSN_COST);
11869 format %{ "ror $dst, $src, $shift" %}
11870
11871 ins_encode %{
11872 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11873 $shift$$constant & 0x1f);
11874 %}
11875 ins_pipe(ialu_reg_reg_vshift);
11876 %}
11877
11878 // This pattern is automatically generated from aarch64_ad.m4
11879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11880 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11881 %{
11882 match(Set dst (RotateRight src shift));
11883
11884 ins_cost(INSN_COST);
11885 format %{ "ror $dst, $src, $shift" %}
11886
11887 ins_encode %{
11888 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11889 $shift$$constant & 0x3f);
11890 %}
11891 ins_pipe(ialu_reg_reg_vshift);
11892 %}
11893
11894 // This pattern is automatically generated from aarch64_ad.m4
11895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11896 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11897 %{
11898 match(Set dst (RotateRight src shift));
11899
11900 ins_cost(INSN_COST);
11901 format %{ "ror $dst, $src, $shift" %}
11902
11903 ins_encode %{
11904 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11905 %}
11906 ins_pipe(ialu_reg_reg_vshift);
11907 %}
11908
11909 // This pattern is automatically generated from aarch64_ad.m4
11910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11911 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11912 %{
11913 match(Set dst (RotateRight src shift));
11914
11915 ins_cost(INSN_COST);
11916 format %{ "ror $dst, $src, $shift" %}
11917
11918 ins_encode %{
11919 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11920 %}
11921 ins_pipe(ialu_reg_reg_vshift);
11922 %}
11923
11924 // This pattern is automatically generated from aarch64_ad.m4
11925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11926 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11927 %{
11928 match(Set dst (RotateLeft src shift));
11929
11930 ins_cost(INSN_COST);
11931 format %{ "rol $dst, $src, $shift" %}
11932
11933 ins_encode %{
11934 __ subw(rscratch1, zr, as_Register($shift$$reg));
11935 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11936 %}
11937 ins_pipe(ialu_reg_reg_vshift);
11938 %}
11939
11940 // This pattern is automatically generated from aarch64_ad.m4
11941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11942 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11943 %{
11944 match(Set dst (RotateLeft src shift));
11945
11946 ins_cost(INSN_COST);
11947 format %{ "rol $dst, $src, $shift" %}
11948
11949 ins_encode %{
11950 __ subw(rscratch1, zr, as_Register($shift$$reg));
11951 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11952 %}
11953 ins_pipe(ialu_reg_reg_vshift);
11954 %}
11955
11956
11957 // Add/subtract (extended)
11958
11959 // This pattern is automatically generated from aarch64_ad.m4
11960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11961 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11962 %{
11963 match(Set dst (AddL src1 (ConvI2L src2)));
11964 ins_cost(INSN_COST);
11965 format %{ "add $dst, $src1, $src2, sxtw" %}
11966
11967 ins_encode %{
11968 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11969 as_Register($src2$$reg), ext::sxtw);
11970 %}
11971 ins_pipe(ialu_reg_reg);
11972 %}
11973
11974 // This pattern is automatically generated from aarch64_ad.m4
11975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11976 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11977 %{
11978 match(Set dst (SubL src1 (ConvI2L src2)));
11979 ins_cost(INSN_COST);
11980 format %{ "sub $dst, $src1, $src2, sxtw" %}
11981
11982 ins_encode %{
11983 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11984 as_Register($src2$$reg), ext::sxtw);
11985 %}
11986 ins_pipe(ialu_reg_reg);
11987 %}
11988
11989 // This pattern is automatically generated from aarch64_ad.m4
11990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11991 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11992 %{
11993 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11994 ins_cost(INSN_COST);
11995 format %{ "add $dst, $src1, $src2, sxth" %}
11996
11997 ins_encode %{
11998 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11999 as_Register($src2$$reg), ext::sxth);
12000 %}
12001 ins_pipe(ialu_reg_reg);
12002 %}
12003
12004 // This pattern is automatically generated from aarch64_ad.m4
12005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12006 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12007 %{
12008 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12009 ins_cost(INSN_COST);
12010 format %{ "add $dst, $src1, $src2, sxtb" %}
12011
12012 ins_encode %{
12013 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12014 as_Register($src2$$reg), ext::sxtb);
12015 %}
12016 ins_pipe(ialu_reg_reg);
12017 %}
12018
12019 // This pattern is automatically generated from aarch64_ad.m4
12020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12021 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12022 %{
12023 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12024 ins_cost(INSN_COST);
12025 format %{ "add $dst, $src1, $src2, uxtb" %}
12026
12027 ins_encode %{
12028 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12029 as_Register($src2$$reg), ext::uxtb);
12030 %}
12031 ins_pipe(ialu_reg_reg);
12032 %}
12033
12034 // This pattern is automatically generated from aarch64_ad.m4
12035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12036 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12037 %{
12038 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12039 ins_cost(INSN_COST);
12040 format %{ "add $dst, $src1, $src2, sxth" %}
12041
12042 ins_encode %{
12043 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12044 as_Register($src2$$reg), ext::sxth);
12045 %}
12046 ins_pipe(ialu_reg_reg);
12047 %}
12048
12049 // This pattern is automatically generated from aarch64_ad.m4
12050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12051 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12052 %{
12053 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12054 ins_cost(INSN_COST);
12055 format %{ "add $dst, $src1, $src2, sxtw" %}
12056
12057 ins_encode %{
12058 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12059 as_Register($src2$$reg), ext::sxtw);
12060 %}
12061 ins_pipe(ialu_reg_reg);
12062 %}
12063
12064 // This pattern is automatically generated from aarch64_ad.m4
12065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12066 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12067 %{
12068 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12069 ins_cost(INSN_COST);
12070 format %{ "add $dst, $src1, $src2, sxtb" %}
12071
12072 ins_encode %{
12073 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12074 as_Register($src2$$reg), ext::sxtb);
12075 %}
12076 ins_pipe(ialu_reg_reg);
12077 %}
12078
12079 // This pattern is automatically generated from aarch64_ad.m4
12080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12081 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12082 %{
12083 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12084 ins_cost(INSN_COST);
12085 format %{ "add $dst, $src1, $src2, uxtb" %}
12086
12087 ins_encode %{
12088 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12089 as_Register($src2$$reg), ext::uxtb);
12090 %}
12091 ins_pipe(ialu_reg_reg);
12092 %}
12093
12094 // This pattern is automatically generated from aarch64_ad.m4
12095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12096 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12097 %{
12098 match(Set dst (AddI src1 (AndI src2 mask)));
12099 ins_cost(INSN_COST);
12100 format %{ "addw $dst, $src1, $src2, uxtb" %}
12101
12102 ins_encode %{
12103 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12104 as_Register($src2$$reg), ext::uxtb);
12105 %}
12106 ins_pipe(ialu_reg_reg);
12107 %}
12108
12109 // This pattern is automatically generated from aarch64_ad.m4
12110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12111 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12112 %{
12113 match(Set dst (AddI src1 (AndI src2 mask)));
12114 ins_cost(INSN_COST);
12115 format %{ "addw $dst, $src1, $src2, uxth" %}
12116
12117 ins_encode %{
12118 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12119 as_Register($src2$$reg), ext::uxth);
12120 %}
12121 ins_pipe(ialu_reg_reg);
12122 %}
12123
12124 // This pattern is automatically generated from aarch64_ad.m4
12125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12126 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12127 %{
12128 match(Set dst (AddL src1 (AndL src2 mask)));
12129 ins_cost(INSN_COST);
12130 format %{ "add $dst, $src1, $src2, uxtb" %}
12131
12132 ins_encode %{
12133 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12134 as_Register($src2$$reg), ext::uxtb);
12135 %}
12136 ins_pipe(ialu_reg_reg);
12137 %}
12138
12139 // This pattern is automatically generated from aarch64_ad.m4
12140 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12141 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12142 %{
12143 match(Set dst (AddL src1 (AndL src2 mask)));
12144 ins_cost(INSN_COST);
12145 format %{ "add $dst, $src1, $src2, uxth" %}
12146
12147 ins_encode %{
12148 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12149 as_Register($src2$$reg), ext::uxth);
12150 %}
12151 ins_pipe(ialu_reg_reg);
12152 %}
12153
12154 // This pattern is automatically generated from aarch64_ad.m4
12155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12156 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12157 %{
12158 match(Set dst (AddL src1 (AndL src2 mask)));
12159 ins_cost(INSN_COST);
12160 format %{ "add $dst, $src1, $src2, uxtw" %}
12161
12162 ins_encode %{
12163 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12164 as_Register($src2$$reg), ext::uxtw);
12165 %}
12166 ins_pipe(ialu_reg_reg);
12167 %}
12168
12169 // This pattern is automatically generated from aarch64_ad.m4
12170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12171 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12172 %{
12173 match(Set dst (SubI src1 (AndI src2 mask)));
12174 ins_cost(INSN_COST);
12175 format %{ "subw $dst, $src1, $src2, uxtb" %}
12176
12177 ins_encode %{
12178 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12179 as_Register($src2$$reg), ext::uxtb);
12180 %}
12181 ins_pipe(ialu_reg_reg);
12182 %}
12183
12184 // This pattern is automatically generated from aarch64_ad.m4
12185 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12186 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12187 %{
12188 match(Set dst (SubI src1 (AndI src2 mask)));
12189 ins_cost(INSN_COST);
12190 format %{ "subw $dst, $src1, $src2, uxth" %}
12191
12192 ins_encode %{
12193 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12194 as_Register($src2$$reg), ext::uxth);
12195 %}
12196 ins_pipe(ialu_reg_reg);
12197 %}
12198
12199 // This pattern is automatically generated from aarch64_ad.m4
12200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12201 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12202 %{
12203 match(Set dst (SubL src1 (AndL src2 mask)));
12204 ins_cost(INSN_COST);
12205 format %{ "sub $dst, $src1, $src2, uxtb" %}
12206
12207 ins_encode %{
12208 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12209 as_Register($src2$$reg), ext::uxtb);
12210 %}
12211 ins_pipe(ialu_reg_reg);
12212 %}
12213
12214 // This pattern is automatically generated from aarch64_ad.m4
12215 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12216 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12217 %{
12218 match(Set dst (SubL src1 (AndL src2 mask)));
12219 ins_cost(INSN_COST);
12220 format %{ "sub $dst, $src1, $src2, uxth" %}
12221
12222 ins_encode %{
12223 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12224 as_Register($src2$$reg), ext::uxth);
12225 %}
12226 ins_pipe(ialu_reg_reg);
12227 %}
12228
12229 // This pattern is automatically generated from aarch64_ad.m4
12230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12231 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12232 %{
12233 match(Set dst (SubL src1 (AndL src2 mask)));
12234 ins_cost(INSN_COST);
12235 format %{ "sub $dst, $src1, $src2, uxtw" %}
12236
12237 ins_encode %{
12238 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12239 as_Register($src2$$reg), ext::uxtw);
12240 %}
12241 ins_pipe(ialu_reg_reg);
12242 %}
12243
12244
12245 // This pattern is automatically generated from aarch64_ad.m4
12246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12247 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12248 %{
12249 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12250 ins_cost(1.9 * INSN_COST);
12251 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12252
12253 ins_encode %{
12254 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12255 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12256 %}
12257 ins_pipe(ialu_reg_reg_shift);
12258 %}
12259
12260 // This pattern is automatically generated from aarch64_ad.m4
12261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12262 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12263 %{
12264 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12265 ins_cost(1.9 * INSN_COST);
12266 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12267
12268 ins_encode %{
12269 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12270 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12271 %}
12272 ins_pipe(ialu_reg_reg_shift);
12273 %}
12274
12275 // This pattern is automatically generated from aarch64_ad.m4
12276 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12277 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12278 %{
12279 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12280 ins_cost(1.9 * INSN_COST);
12281 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12282
12283 ins_encode %{
12284 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12285 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12286 %}
12287 ins_pipe(ialu_reg_reg_shift);
12288 %}
12289
12290 // This pattern is automatically generated from aarch64_ad.m4
12291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12292 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12293 %{
12294 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12295 ins_cost(1.9 * INSN_COST);
12296 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12297
12298 ins_encode %{
12299 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12300 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12301 %}
12302 ins_pipe(ialu_reg_reg_shift);
12303 %}
12304
12305 // This pattern is automatically generated from aarch64_ad.m4
12306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12307 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12308 %{
12309 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12310 ins_cost(1.9 * INSN_COST);
12311 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12312
12313 ins_encode %{
12314 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12315 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12316 %}
12317 ins_pipe(ialu_reg_reg_shift);
12318 %}
12319
12320 // This pattern is automatically generated from aarch64_ad.m4
12321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12322 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12323 %{
12324 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12325 ins_cost(1.9 * INSN_COST);
12326 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12327
12328 ins_encode %{
12329 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12330 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12331 %}
12332 ins_pipe(ialu_reg_reg_shift);
12333 %}
12334
12335 // This pattern is automatically generated from aarch64_ad.m4
12336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12337 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12338 %{
12339 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12340 ins_cost(1.9 * INSN_COST);
12341 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12342
12343 ins_encode %{
12344 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12345 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12346 %}
12347 ins_pipe(ialu_reg_reg_shift);
12348 %}
12349
12350 // This pattern is automatically generated from aarch64_ad.m4
12351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12352 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12353 %{
12354 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12355 ins_cost(1.9 * INSN_COST);
12356 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12357
12358 ins_encode %{
12359 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12360 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12361 %}
12362 ins_pipe(ialu_reg_reg_shift);
12363 %}
12364
12365 // This pattern is automatically generated from aarch64_ad.m4
12366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12367 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12368 %{
12369 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12370 ins_cost(1.9 * INSN_COST);
12371 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12372
12373 ins_encode %{
12374 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12375 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12376 %}
12377 ins_pipe(ialu_reg_reg_shift);
12378 %}
12379
12380 // This pattern is automatically generated from aarch64_ad.m4
12381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12382 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12383 %{
12384 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12385 ins_cost(1.9 * INSN_COST);
12386 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12387
12388 ins_encode %{
12389 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12390 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12391 %}
12392 ins_pipe(ialu_reg_reg_shift);
12393 %}
12394
12395 // This pattern is automatically generated from aarch64_ad.m4
12396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12397 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12398 %{
12399 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12400 ins_cost(1.9 * INSN_COST);
12401 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12402
12403 ins_encode %{
12404 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12405 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12406 %}
12407 ins_pipe(ialu_reg_reg_shift);
12408 %}
12409
12410 // This pattern is automatically generated from aarch64_ad.m4
12411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12412 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12413 %{
12414 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12415 ins_cost(1.9 * INSN_COST);
12416 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12417
12418 ins_encode %{
12419 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12420 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12421 %}
12422 ins_pipe(ialu_reg_reg_shift);
12423 %}
12424
12425 // This pattern is automatically generated from aarch64_ad.m4
12426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12427 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12428 %{
12429 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12430 ins_cost(1.9 * INSN_COST);
12431 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12432
12433 ins_encode %{
12434 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12435 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12436 %}
12437 ins_pipe(ialu_reg_reg_shift);
12438 %}
12439
12440 // This pattern is automatically generated from aarch64_ad.m4
12441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12442 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12443 %{
12444 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12445 ins_cost(1.9 * INSN_COST);
12446 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12447
12448 ins_encode %{
12449 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12450 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12451 %}
12452 ins_pipe(ialu_reg_reg_shift);
12453 %}
12454
12455 // This pattern is automatically generated from aarch64_ad.m4
12456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12457 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12458 %{
12459 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12460 ins_cost(1.9 * INSN_COST);
12461 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12462
12463 ins_encode %{
12464 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12465 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12466 %}
12467 ins_pipe(ialu_reg_reg_shift);
12468 %}
12469
12470 // This pattern is automatically generated from aarch64_ad.m4
12471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12472 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12473 %{
12474 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12475 ins_cost(1.9 * INSN_COST);
12476 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12477
12478 ins_encode %{
12479 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12480 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12481 %}
12482 ins_pipe(ialu_reg_reg_shift);
12483 %}
12484
12485 // This pattern is automatically generated from aarch64_ad.m4
12486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12487 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12488 %{
12489 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12490 ins_cost(1.9 * INSN_COST);
12491 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12492
12493 ins_encode %{
12494 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12495 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12496 %}
12497 ins_pipe(ialu_reg_reg_shift);
12498 %}
12499
12500 // This pattern is automatically generated from aarch64_ad.m4
12501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12502 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12503 %{
12504 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12505 ins_cost(1.9 * INSN_COST);
12506 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12507
12508 ins_encode %{
12509 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12510 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12511 %}
12512 ins_pipe(ialu_reg_reg_shift);
12513 %}
12514
12515 // This pattern is automatically generated from aarch64_ad.m4
12516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12517 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12518 %{
12519 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12520 ins_cost(1.9 * INSN_COST);
12521 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12522
12523 ins_encode %{
12524 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12525 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12526 %}
12527 ins_pipe(ialu_reg_reg_shift);
12528 %}
12529
12530 // This pattern is automatically generated from aarch64_ad.m4
12531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12532 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12533 %{
12534 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12535 ins_cost(1.9 * INSN_COST);
12536 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12537
12538 ins_encode %{
12539 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12540 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12541 %}
12542 ins_pipe(ialu_reg_reg_shift);
12543 %}
12544
12545 // This pattern is automatically generated from aarch64_ad.m4
12546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12547 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12548 %{
12549 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12550 ins_cost(1.9 * INSN_COST);
12551 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12552
12553 ins_encode %{
12554 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12555 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12556 %}
12557 ins_pipe(ialu_reg_reg_shift);
12558 %}
12559
12560 // This pattern is automatically generated from aarch64_ad.m4
12561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12562 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12563 %{
12564 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12565 ins_cost(1.9 * INSN_COST);
12566 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12567
12568 ins_encode %{
12569 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12570 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12571 %}
12572 ins_pipe(ialu_reg_reg_shift);
12573 %}
12574
12575 // This pattern is automatically generated from aarch64_ad.m4
12576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12577 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12578 %{
12579 effect(DEF dst, USE src1, USE src2, USE cr);
12580 ins_cost(INSN_COST * 2);
12581 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12582
12583 ins_encode %{
12584 __ cselw($dst$$Register,
12585 $src1$$Register,
12586 $src2$$Register,
12587 Assembler::LT);
12588 %}
12589 ins_pipe(icond_reg_reg);
12590 %}
12591
12592 // This pattern is automatically generated from aarch64_ad.m4
12593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12594 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12595 %{
12596 effect(DEF dst, USE src1, USE src2, USE cr);
12597 ins_cost(INSN_COST * 2);
12598 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12599
12600 ins_encode %{
12601 __ cselw($dst$$Register,
12602 $src1$$Register,
12603 $src2$$Register,
12604 Assembler::GT);
12605 %}
12606 ins_pipe(icond_reg_reg);
12607 %}
12608
12609 // This pattern is automatically generated from aarch64_ad.m4
12610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12611 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12612 %{
12613 effect(DEF dst, USE src1, USE cr);
12614 ins_cost(INSN_COST * 2);
12615 format %{ "cselw $dst, $src1, zr lt\t" %}
12616
12617 ins_encode %{
12618 __ cselw($dst$$Register,
12619 $src1$$Register,
12620 zr,
12621 Assembler::LT);
12622 %}
12623 ins_pipe(icond_reg);
12624 %}
12625
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12629 %{
12630 effect(DEF dst, USE src1, USE cr);
12631 ins_cost(INSN_COST * 2);
12632 format %{ "cselw $dst, $src1, zr gt\t" %}
12633
12634 ins_encode %{
12635 __ cselw($dst$$Register,
12636 $src1$$Register,
12637 zr,
12638 Assembler::GT);
12639 %}
12640 ins_pipe(icond_reg);
12641 %}
12642
12643 // This pattern is automatically generated from aarch64_ad.m4
12644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12645 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12646 %{
12647 effect(DEF dst, USE src1, USE cr);
12648 ins_cost(INSN_COST * 2);
12649 format %{ "csincw $dst, $src1, zr le\t" %}
12650
12651 ins_encode %{
12652 __ csincw($dst$$Register,
12653 $src1$$Register,
12654 zr,
12655 Assembler::LE);
12656 %}
12657 ins_pipe(icond_reg);
12658 %}
12659
12660 // This pattern is automatically generated from aarch64_ad.m4
12661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12662 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12663 %{
12664 effect(DEF dst, USE src1, USE cr);
12665 ins_cost(INSN_COST * 2);
12666 format %{ "csincw $dst, $src1, zr gt\t" %}
12667
12668 ins_encode %{
12669 __ csincw($dst$$Register,
12670 $src1$$Register,
12671 zr,
12672 Assembler::GT);
12673 %}
12674 ins_pipe(icond_reg);
12675 %}
12676
12677 // This pattern is automatically generated from aarch64_ad.m4
12678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12679 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12680 %{
12681 effect(DEF dst, USE src1, USE cr);
12682 ins_cost(INSN_COST * 2);
12683 format %{ "csinvw $dst, $src1, zr lt\t" %}
12684
12685 ins_encode %{
12686 __ csinvw($dst$$Register,
12687 $src1$$Register,
12688 zr,
12689 Assembler::LT);
12690 %}
12691 ins_pipe(icond_reg);
12692 %}
12693
12694 // This pattern is automatically generated from aarch64_ad.m4
12695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12696 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12697 %{
12698 effect(DEF dst, USE src1, USE cr);
12699 ins_cost(INSN_COST * 2);
12700 format %{ "csinvw $dst, $src1, zr ge\t" %}
12701
12702 ins_encode %{
12703 __ csinvw($dst$$Register,
12704 $src1$$Register,
12705 zr,
12706 Assembler::GE);
12707 %}
12708 ins_pipe(icond_reg);
12709 %}
12710
12711 // This pattern is automatically generated from aarch64_ad.m4
12712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12713 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12714 %{
12715 match(Set dst (MinI src imm));
12716 ins_cost(INSN_COST * 3);
12717 expand %{
12718 rFlagsReg cr;
12719 compI_reg_imm0(cr, src);
12720 cmovI_reg_imm0_lt(dst, src, cr);
12721 %}
12722 %}
12723
12724 // This pattern is automatically generated from aarch64_ad.m4
12725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12726 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12727 %{
12728 match(Set dst (MinI imm src));
12729 ins_cost(INSN_COST * 3);
12730 expand %{
12731 rFlagsReg cr;
12732 compI_reg_imm0(cr, src);
12733 cmovI_reg_imm0_lt(dst, src, cr);
12734 %}
12735 %}
12736
12737 // This pattern is automatically generated from aarch64_ad.m4
12738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12739 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12740 %{
12741 match(Set dst (MinI src imm));
12742 ins_cost(INSN_COST * 3);
12743 expand %{
12744 rFlagsReg cr;
12745 compI_reg_imm0(cr, src);
12746 cmovI_reg_imm1_le(dst, src, cr);
12747 %}
12748 %}
12749
12750 // This pattern is automatically generated from aarch64_ad.m4
12751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12752 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12753 %{
12754 match(Set dst (MinI imm src));
12755 ins_cost(INSN_COST * 3);
12756 expand %{
12757 rFlagsReg cr;
12758 compI_reg_imm0(cr, src);
12759 cmovI_reg_imm1_le(dst, src, cr);
12760 %}
12761 %}
12762
12763 // This pattern is automatically generated from aarch64_ad.m4
12764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12765 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12766 %{
12767 match(Set dst (MinI src imm));
12768 ins_cost(INSN_COST * 3);
12769 expand %{
12770 rFlagsReg cr;
12771 compI_reg_imm0(cr, src);
12772 cmovI_reg_immM1_lt(dst, src, cr);
12773 %}
12774 %}
12775
12776 // This pattern is automatically generated from aarch64_ad.m4
12777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12778 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12779 %{
12780 match(Set dst (MinI imm src));
12781 ins_cost(INSN_COST * 3);
12782 expand %{
12783 rFlagsReg cr;
12784 compI_reg_imm0(cr, src);
12785 cmovI_reg_immM1_lt(dst, src, cr);
12786 %}
12787 %}
12788
12789 // This pattern is automatically generated from aarch64_ad.m4
12790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12791 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12792 %{
12793 match(Set dst (MaxI src imm));
12794 ins_cost(INSN_COST * 3);
12795 expand %{
12796 rFlagsReg cr;
12797 compI_reg_imm0(cr, src);
12798 cmovI_reg_imm0_gt(dst, src, cr);
12799 %}
12800 %}
12801
12802 // This pattern is automatically generated from aarch64_ad.m4
12803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12804 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12805 %{
12806 match(Set dst (MaxI imm src));
12807 ins_cost(INSN_COST * 3);
12808 expand %{
12809 rFlagsReg cr;
12810 compI_reg_imm0(cr, src);
12811 cmovI_reg_imm0_gt(dst, src, cr);
12812 %}
12813 %}
12814
12815 // This pattern is automatically generated from aarch64_ad.m4
12816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12817 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12818 %{
12819 match(Set dst (MaxI src imm));
12820 ins_cost(INSN_COST * 3);
12821 expand %{
12822 rFlagsReg cr;
12823 compI_reg_imm0(cr, src);
12824 cmovI_reg_imm1_gt(dst, src, cr);
12825 %}
12826 %}
12827
12828 // This pattern is automatically generated from aarch64_ad.m4
12829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12830 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12831 %{
12832 match(Set dst (MaxI imm src));
12833 ins_cost(INSN_COST * 3);
12834 expand %{
12835 rFlagsReg cr;
12836 compI_reg_imm0(cr, src);
12837 cmovI_reg_imm1_gt(dst, src, cr);
12838 %}
12839 %}
12840
12841 // This pattern is automatically generated from aarch64_ad.m4
12842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12843 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12844 %{
12845 match(Set dst (MaxI src imm));
12846 ins_cost(INSN_COST * 3);
12847 expand %{
12848 rFlagsReg cr;
12849 compI_reg_imm0(cr, src);
12850 cmovI_reg_immM1_ge(dst, src, cr);
12851 %}
12852 %}
12853
12854 // This pattern is automatically generated from aarch64_ad.m4
12855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12856 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12857 %{
12858 match(Set dst (MaxI imm src));
12859 ins_cost(INSN_COST * 3);
12860 expand %{
12861 rFlagsReg cr;
12862 compI_reg_imm0(cr, src);
12863 cmovI_reg_immM1_ge(dst, src, cr);
12864 %}
12865 %}
12866
12867 // This pattern is automatically generated from aarch64_ad.m4
12868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12869 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12870 %{
12871 match(Set dst (ReverseI src));
12872 ins_cost(INSN_COST);
12873 format %{ "rbitw $dst, $src" %}
12874 ins_encode %{
12875 __ rbitw($dst$$Register, $src$$Register);
12876 %}
12877 ins_pipe(ialu_reg);
12878 %}
12879
12880 // This pattern is automatically generated from aarch64_ad.m4
12881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12882 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12883 %{
12884 match(Set dst (ReverseL src));
12885 ins_cost(INSN_COST);
12886 format %{ "rbit $dst, $src" %}
12887 ins_encode %{
12888 __ rbit($dst$$Register, $src$$Register);
12889 %}
12890 ins_pipe(ialu_reg);
12891 %}
12892
12893
12894 // END This section of the file is automatically generated. Do not edit --------------
12895
12896
12897 // ============================================================================
12898 // Floating Point Arithmetic Instructions
12899
12900 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12901 match(Set dst (AddHF src1 src2));
12902 format %{ "faddh $dst, $src1, $src2" %}
12903 ins_encode %{
12904 __ faddh($dst$$FloatRegister,
12905 $src1$$FloatRegister,
12906 $src2$$FloatRegister);
12907 %}
12908 ins_pipe(fp_dop_reg_reg_s);
12909 %}
12910
12911 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12912 match(Set dst (AddF src1 src2));
12913
12914 ins_cost(INSN_COST * 5);
12915 format %{ "fadds $dst, $src1, $src2" %}
12916
12917 ins_encode %{
12918 __ fadds(as_FloatRegister($dst$$reg),
12919 as_FloatRegister($src1$$reg),
12920 as_FloatRegister($src2$$reg));
12921 %}
12922
12923 ins_pipe(fp_dop_reg_reg_s);
12924 %}
12925
12926 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12927 match(Set dst (AddD src1 src2));
12928
12929 ins_cost(INSN_COST * 5);
12930 format %{ "faddd $dst, $src1, $src2" %}
12931
12932 ins_encode %{
12933 __ faddd(as_FloatRegister($dst$$reg),
12934 as_FloatRegister($src1$$reg),
12935 as_FloatRegister($src2$$reg));
12936 %}
12937
12938 ins_pipe(fp_dop_reg_reg_d);
12939 %}
12940
12941 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12942 match(Set dst (SubHF src1 src2));
12943 format %{ "fsubh $dst, $src1, $src2" %}
12944 ins_encode %{
12945 __ fsubh($dst$$FloatRegister,
12946 $src1$$FloatRegister,
12947 $src2$$FloatRegister);
12948 %}
12949 ins_pipe(fp_dop_reg_reg_s);
12950 %}
12951
12952 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12953 match(Set dst (SubF src1 src2));
12954
12955 ins_cost(INSN_COST * 5);
12956 format %{ "fsubs $dst, $src1, $src2" %}
12957
12958 ins_encode %{
12959 __ fsubs(as_FloatRegister($dst$$reg),
12960 as_FloatRegister($src1$$reg),
12961 as_FloatRegister($src2$$reg));
12962 %}
12963
12964 ins_pipe(fp_dop_reg_reg_s);
12965 %}
12966
12967 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12968 match(Set dst (SubD src1 src2));
12969
12970 ins_cost(INSN_COST * 5);
12971 format %{ "fsubd $dst, $src1, $src2" %}
12972
12973 ins_encode %{
12974 __ fsubd(as_FloatRegister($dst$$reg),
12975 as_FloatRegister($src1$$reg),
12976 as_FloatRegister($src2$$reg));
12977 %}
12978
12979 ins_pipe(fp_dop_reg_reg_d);
12980 %}
12981
12982 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12983 match(Set dst (MulHF src1 src2));
12984 format %{ "fmulh $dst, $src1, $src2" %}
12985 ins_encode %{
12986 __ fmulh($dst$$FloatRegister,
12987 $src1$$FloatRegister,
12988 $src2$$FloatRegister);
12989 %}
12990 ins_pipe(fp_dop_reg_reg_s);
12991 %}
12992
12993 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12994 match(Set dst (MulF src1 src2));
12995
12996 ins_cost(INSN_COST * 6);
12997 format %{ "fmuls $dst, $src1, $src2" %}
12998
12999 ins_encode %{
13000 __ fmuls(as_FloatRegister($dst$$reg),
13001 as_FloatRegister($src1$$reg),
13002 as_FloatRegister($src2$$reg));
13003 %}
13004
13005 ins_pipe(fp_dop_reg_reg_s);
13006 %}
13007
13008 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13009 match(Set dst (MulD src1 src2));
13010
13011 ins_cost(INSN_COST * 6);
13012 format %{ "fmuld $dst, $src1, $src2" %}
13013
13014 ins_encode %{
13015 __ fmuld(as_FloatRegister($dst$$reg),
13016 as_FloatRegister($src1$$reg),
13017 as_FloatRegister($src2$$reg));
13018 %}
13019
13020 ins_pipe(fp_dop_reg_reg_d);
13021 %}
13022
13023 // src1 * src2 + src3 (half-precision float)
13024 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13025 match(Set dst (FmaHF src3 (Binary src1 src2)));
13026 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13027 ins_encode %{
13028 assert(UseFMA, "Needs FMA instructions support.");
13029 __ fmaddh($dst$$FloatRegister,
13030 $src1$$FloatRegister,
13031 $src2$$FloatRegister,
13032 $src3$$FloatRegister);
13033 %}
13034 ins_pipe(pipe_class_default);
13035 %}
13036
13037 // src1 * src2 + src3
13038 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13039 match(Set dst (FmaF src3 (Binary src1 src2)));
13040
13041 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13042
13043 ins_encode %{
13044 assert(UseFMA, "Needs FMA instructions support.");
13045 __ fmadds(as_FloatRegister($dst$$reg),
13046 as_FloatRegister($src1$$reg),
13047 as_FloatRegister($src2$$reg),
13048 as_FloatRegister($src3$$reg));
13049 %}
13050
13051 ins_pipe(pipe_class_default);
13052 %}
13053
13054 // src1 * src2 + src3
13055 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13056 match(Set dst (FmaD src3 (Binary src1 src2)));
13057
13058 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13059
13060 ins_encode %{
13061 assert(UseFMA, "Needs FMA instructions support.");
13062 __ fmaddd(as_FloatRegister($dst$$reg),
13063 as_FloatRegister($src1$$reg),
13064 as_FloatRegister($src2$$reg),
13065 as_FloatRegister($src3$$reg));
13066 %}
13067
13068 ins_pipe(pipe_class_default);
13069 %}
13070
13071 // src1 * (-src2) + src3
13072 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13073 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13074 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13075
13076 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13077
13078 ins_encode %{
13079 assert(UseFMA, "Needs FMA instructions support.");
13080 __ fmsubs(as_FloatRegister($dst$$reg),
13081 as_FloatRegister($src1$$reg),
13082 as_FloatRegister($src2$$reg),
13083 as_FloatRegister($src3$$reg));
13084 %}
13085
13086 ins_pipe(pipe_class_default);
13087 %}
13088
13089 // src1 * (-src2) + src3
13090 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13091 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13092 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13093
13094 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13095
13096 ins_encode %{
13097 assert(UseFMA, "Needs FMA instructions support.");
13098 __ fmsubd(as_FloatRegister($dst$$reg),
13099 as_FloatRegister($src1$$reg),
13100 as_FloatRegister($src2$$reg),
13101 as_FloatRegister($src3$$reg));
13102 %}
13103
13104 ins_pipe(pipe_class_default);
13105 %}
13106
13107 // src1 * (-src2) - src3
13108 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13109 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13110 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13111
13112 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13113
13114 ins_encode %{
13115 assert(UseFMA, "Needs FMA instructions support.");
13116 __ fnmadds(as_FloatRegister($dst$$reg),
13117 as_FloatRegister($src1$$reg),
13118 as_FloatRegister($src2$$reg),
13119 as_FloatRegister($src3$$reg));
13120 %}
13121
13122 ins_pipe(pipe_class_default);
13123 %}
13124
13125 // src1 * (-src2) - src3
13126 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13127 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13128 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13129
13130 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13131
13132 ins_encode %{
13133 assert(UseFMA, "Needs FMA instructions support.");
13134 __ fnmaddd(as_FloatRegister($dst$$reg),
13135 as_FloatRegister($src1$$reg),
13136 as_FloatRegister($src2$$reg),
13137 as_FloatRegister($src3$$reg));
13138 %}
13139
13140 ins_pipe(pipe_class_default);
13141 %}
13142
13143 // src1 * src2 - src3
13144 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13145 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13146
13147 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13148
13149 ins_encode %{
13150 assert(UseFMA, "Needs FMA instructions support.");
13151 __ fnmsubs(as_FloatRegister($dst$$reg),
13152 as_FloatRegister($src1$$reg),
13153 as_FloatRegister($src2$$reg),
13154 as_FloatRegister($src3$$reg));
13155 %}
13156
13157 ins_pipe(pipe_class_default);
13158 %}
13159
13160 // src1 * src2 - src3
13161 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13162 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13163
13164 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13165
13166 ins_encode %{
13167 assert(UseFMA, "Needs FMA instructions support.");
13168 // n.b. insn name should be fnmsubd
13169 __ fnmsub(as_FloatRegister($dst$$reg),
13170 as_FloatRegister($src1$$reg),
13171 as_FloatRegister($src2$$reg),
13172 as_FloatRegister($src3$$reg));
13173 %}
13174
13175 ins_pipe(pipe_class_default);
13176 %}
13177
13178 // Math.max(HH)H (half-precision float)
13179 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13180 match(Set dst (MaxHF src1 src2));
13181 format %{ "fmaxh $dst, $src1, $src2" %}
13182 ins_encode %{
13183 __ fmaxh($dst$$FloatRegister,
13184 $src1$$FloatRegister,
13185 $src2$$FloatRegister);
13186 %}
13187 ins_pipe(fp_dop_reg_reg_s);
13188 %}
13189
13190 // Math.min(HH)H (half-precision float)
13191 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13192 match(Set dst (MinHF src1 src2));
13193 format %{ "fminh $dst, $src1, $src2" %}
13194 ins_encode %{
13195 __ fminh($dst$$FloatRegister,
13196 $src1$$FloatRegister,
13197 $src2$$FloatRegister);
13198 %}
13199 ins_pipe(fp_dop_reg_reg_s);
13200 %}
13201
13202 // Math.max(FF)F
13203 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13204 match(Set dst (MaxF src1 src2));
13205
13206 format %{ "fmaxs $dst, $src1, $src2" %}
13207 ins_encode %{
13208 __ fmaxs(as_FloatRegister($dst$$reg),
13209 as_FloatRegister($src1$$reg),
13210 as_FloatRegister($src2$$reg));
13211 %}
13212
13213 ins_pipe(fp_dop_reg_reg_s);
13214 %}
13215
13216 // Math.min(FF)F
13217 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13218 match(Set dst (MinF src1 src2));
13219
13220 format %{ "fmins $dst, $src1, $src2" %}
13221 ins_encode %{
13222 __ fmins(as_FloatRegister($dst$$reg),
13223 as_FloatRegister($src1$$reg),
13224 as_FloatRegister($src2$$reg));
13225 %}
13226
13227 ins_pipe(fp_dop_reg_reg_s);
13228 %}
13229
13230 // Math.max(DD)D
13231 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13232 match(Set dst (MaxD src1 src2));
13233
13234 format %{ "fmaxd $dst, $src1, $src2" %}
13235 ins_encode %{
13236 __ fmaxd(as_FloatRegister($dst$$reg),
13237 as_FloatRegister($src1$$reg),
13238 as_FloatRegister($src2$$reg));
13239 %}
13240
13241 ins_pipe(fp_dop_reg_reg_d);
13242 %}
13243
13244 // Math.min(DD)D
13245 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13246 match(Set dst (MinD src1 src2));
13247
13248 format %{ "fmind $dst, $src1, $src2" %}
13249 ins_encode %{
13250 __ fmind(as_FloatRegister($dst$$reg),
13251 as_FloatRegister($src1$$reg),
13252 as_FloatRegister($src2$$reg));
13253 %}
13254
13255 ins_pipe(fp_dop_reg_reg_d);
13256 %}
13257
13258 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13259 match(Set dst (DivHF src1 src2));
13260 format %{ "fdivh $dst, $src1, $src2" %}
13261 ins_encode %{
13262 __ fdivh($dst$$FloatRegister,
13263 $src1$$FloatRegister,
13264 $src2$$FloatRegister);
13265 %}
13266 ins_pipe(fp_div_s);
13267 %}
13268
13269 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13270 match(Set dst (DivF src1 src2));
13271
13272 ins_cost(INSN_COST * 18);
13273 format %{ "fdivs $dst, $src1, $src2" %}
13274
13275 ins_encode %{
13276 __ fdivs(as_FloatRegister($dst$$reg),
13277 as_FloatRegister($src1$$reg),
13278 as_FloatRegister($src2$$reg));
13279 %}
13280
13281 ins_pipe(fp_div_s);
13282 %}
13283
13284 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13285 match(Set dst (DivD src1 src2));
13286
13287 ins_cost(INSN_COST * 32);
13288 format %{ "fdivd $dst, $src1, $src2" %}
13289
13290 ins_encode %{
13291 __ fdivd(as_FloatRegister($dst$$reg),
13292 as_FloatRegister($src1$$reg),
13293 as_FloatRegister($src2$$reg));
13294 %}
13295
13296 ins_pipe(fp_div_d);
13297 %}
13298
13299 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13300 match(Set dst (NegF src));
13301
13302 ins_cost(INSN_COST * 3);
13303 format %{ "fneg $dst, $src" %}
13304
13305 ins_encode %{
13306 __ fnegs(as_FloatRegister($dst$$reg),
13307 as_FloatRegister($src$$reg));
13308 %}
13309
13310 ins_pipe(fp_uop_s);
13311 %}
13312
13313 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13314 match(Set dst (NegD src));
13315
13316 ins_cost(INSN_COST * 3);
13317 format %{ "fnegd $dst, $src" %}
13318
13319 ins_encode %{
13320 __ fnegd(as_FloatRegister($dst$$reg),
13321 as_FloatRegister($src$$reg));
13322 %}
13323
13324 ins_pipe(fp_uop_d);
13325 %}
13326
13327 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13328 %{
13329 match(Set dst (AbsI src));
13330
13331 effect(KILL cr);
13332 ins_cost(INSN_COST * 2);
13333 format %{ "cmpw $src, zr\n\t"
13334 "cnegw $dst, $src, Assembler::LT\t# int abs"
13335 %}
13336
13337 ins_encode %{
13338 __ cmpw(as_Register($src$$reg), zr);
13339 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13340 %}
13341 ins_pipe(pipe_class_default);
13342 %}
13343
13344 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13345 %{
13346 match(Set dst (AbsL src));
13347
13348 effect(KILL cr);
13349 ins_cost(INSN_COST * 2);
13350 format %{ "cmp $src, zr\n\t"
13351 "cneg $dst, $src, Assembler::LT\t# long abs"
13352 %}
13353
13354 ins_encode %{
13355 __ cmp(as_Register($src$$reg), zr);
13356 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13357 %}
13358 ins_pipe(pipe_class_default);
13359 %}
13360
13361 instruct absF_reg(vRegF dst, vRegF src) %{
13362 match(Set dst (AbsF src));
13363
13364 ins_cost(INSN_COST * 3);
13365 format %{ "fabss $dst, $src" %}
13366 ins_encode %{
13367 __ fabss(as_FloatRegister($dst$$reg),
13368 as_FloatRegister($src$$reg));
13369 %}
13370
13371 ins_pipe(fp_uop_s);
13372 %}
13373
13374 instruct absD_reg(vRegD dst, vRegD src) %{
13375 match(Set dst (AbsD src));
13376
13377 ins_cost(INSN_COST * 3);
13378 format %{ "fabsd $dst, $src" %}
13379 ins_encode %{
13380 __ fabsd(as_FloatRegister($dst$$reg),
13381 as_FloatRegister($src$$reg));
13382 %}
13383
13384 ins_pipe(fp_uop_d);
13385 %}
13386
13387 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13388 match(Set dst (AbsF (SubF src1 src2)));
13389
13390 ins_cost(INSN_COST * 3);
13391 format %{ "fabds $dst, $src1, $src2" %}
13392 ins_encode %{
13393 __ fabds(as_FloatRegister($dst$$reg),
13394 as_FloatRegister($src1$$reg),
13395 as_FloatRegister($src2$$reg));
13396 %}
13397
13398 ins_pipe(fp_uop_s);
13399 %}
13400
13401 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13402 match(Set dst (AbsD (SubD src1 src2)));
13403
13404 ins_cost(INSN_COST * 3);
13405 format %{ "fabdd $dst, $src1, $src2" %}
13406 ins_encode %{
13407 __ fabdd(as_FloatRegister($dst$$reg),
13408 as_FloatRegister($src1$$reg),
13409 as_FloatRegister($src2$$reg));
13410 %}
13411
13412 ins_pipe(fp_uop_d);
13413 %}
13414
13415 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13416 match(Set dst (SqrtD src));
13417
13418 ins_cost(INSN_COST * 50);
13419 format %{ "fsqrtd $dst, $src" %}
13420 ins_encode %{
13421 __ fsqrtd(as_FloatRegister($dst$$reg),
13422 as_FloatRegister($src$$reg));
13423 %}
13424
13425 ins_pipe(fp_div_s);
13426 %}
13427
13428 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13429 match(Set dst (SqrtF src));
13430
13431 ins_cost(INSN_COST * 50);
13432 format %{ "fsqrts $dst, $src" %}
13433 ins_encode %{
13434 __ fsqrts(as_FloatRegister($dst$$reg),
13435 as_FloatRegister($src$$reg));
13436 %}
13437
13438 ins_pipe(fp_div_d);
13439 %}
13440
13441 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13442 match(Set dst (SqrtHF src));
13443 format %{ "fsqrth $dst, $src" %}
13444 ins_encode %{
13445 __ fsqrth($dst$$FloatRegister,
13446 $src$$FloatRegister);
13447 %}
13448 ins_pipe(fp_div_s);
13449 %}
13450
13451 // Math.rint, floor, ceil
13452 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13453 match(Set dst (RoundDoubleMode src rmode));
13454 format %{ "frint $dst, $src, $rmode" %}
13455 ins_encode %{
13456 switch ($rmode$$constant) {
13457 case RoundDoubleModeNode::rmode_rint:
13458 __ frintnd(as_FloatRegister($dst$$reg),
13459 as_FloatRegister($src$$reg));
13460 break;
13461 case RoundDoubleModeNode::rmode_floor:
13462 __ frintmd(as_FloatRegister($dst$$reg),
13463 as_FloatRegister($src$$reg));
13464 break;
13465 case RoundDoubleModeNode::rmode_ceil:
13466 __ frintpd(as_FloatRegister($dst$$reg),
13467 as_FloatRegister($src$$reg));
13468 break;
13469 }
13470 %}
13471 ins_pipe(fp_uop_d);
13472 %}
13473
13474 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13475 match(Set dst (CopySignD src1 (Binary src2 zero)));
13476 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13477 format %{ "CopySignD $dst $src1 $src2" %}
13478 ins_encode %{
13479 FloatRegister dst = as_FloatRegister($dst$$reg),
13480 src1 = as_FloatRegister($src1$$reg),
13481 src2 = as_FloatRegister($src2$$reg),
13482 zero = as_FloatRegister($zero$$reg);
13483 __ fnegd(dst, zero);
13484 __ bsl(dst, __ T8B, src2, src1);
13485 %}
13486 ins_pipe(fp_uop_d);
13487 %}
13488
13489 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13490 match(Set dst (CopySignF src1 src2));
13491 effect(TEMP_DEF dst, USE src1, USE src2);
13492 format %{ "CopySignF $dst $src1 $src2" %}
13493 ins_encode %{
13494 FloatRegister dst = as_FloatRegister($dst$$reg),
13495 src1 = as_FloatRegister($src1$$reg),
13496 src2 = as_FloatRegister($src2$$reg);
13497 __ movi(dst, __ T2S, 0x80, 24);
13498 __ bsl(dst, __ T8B, src2, src1);
13499 %}
13500 ins_pipe(fp_uop_d);
13501 %}
13502
13503 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13504 match(Set dst (SignumD src (Binary zero one)));
13505 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13506 format %{ "signumD $dst, $src" %}
13507 ins_encode %{
13508 FloatRegister src = as_FloatRegister($src$$reg),
13509 dst = as_FloatRegister($dst$$reg),
13510 zero = as_FloatRegister($zero$$reg),
13511 one = as_FloatRegister($one$$reg);
13512 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13513 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13514 // Bit selection instruction gets bit from "one" for each enabled bit in
13515 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13516 // NaN the whole "src" will be copied because "dst" is zero. For all other
13517 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13518 // from "src", and all other bits are copied from 1.0.
13519 __ bsl(dst, __ T8B, one, src);
13520 %}
13521 ins_pipe(fp_uop_d);
13522 %}
13523
13524 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13525 match(Set dst (SignumF src (Binary zero one)));
13526 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13527 format %{ "signumF $dst, $src" %}
13528 ins_encode %{
13529 FloatRegister src = as_FloatRegister($src$$reg),
13530 dst = as_FloatRegister($dst$$reg),
13531 zero = as_FloatRegister($zero$$reg),
13532 one = as_FloatRegister($one$$reg);
13533 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13534 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13535 // Bit selection instruction gets bit from "one" for each enabled bit in
13536 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13537 // NaN the whole "src" will be copied because "dst" is zero. For all other
13538 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13539 // from "src", and all other bits are copied from 1.0.
13540 __ bsl(dst, __ T8B, one, src);
13541 %}
13542 ins_pipe(fp_uop_d);
13543 %}
13544
13545 instruct onspinwait() %{
13546 match(OnSpinWait);
13547 ins_cost(INSN_COST);
13548
13549 format %{ "onspinwait" %}
13550
13551 ins_encode %{
13552 __ spin_wait();
13553 %}
13554 ins_pipe(pipe_class_empty);
13555 %}
13556
13557 // ============================================================================
13558 // Logical Instructions
13559
13560 // Integer Logical Instructions
13561
13562 // And Instructions
13563
13564
13565 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13566 match(Set dst (AndI src1 src2));
13567
13568 format %{ "andw $dst, $src1, $src2\t# int" %}
13569
13570 ins_cost(INSN_COST);
13571 ins_encode %{
13572 __ andw(as_Register($dst$$reg),
13573 as_Register($src1$$reg),
13574 as_Register($src2$$reg));
13575 %}
13576
13577 ins_pipe(ialu_reg_reg);
13578 %}
13579
13580 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13581 match(Set dst (AndI src1 src2));
13582
13583 format %{ "andsw $dst, $src1, $src2\t# int" %}
13584
13585 ins_cost(INSN_COST);
13586 ins_encode %{
13587 __ andw(as_Register($dst$$reg),
13588 as_Register($src1$$reg),
13589 (uint64_t)($src2$$constant));
13590 %}
13591
13592 ins_pipe(ialu_reg_imm);
13593 %}
13594
13595 // Or Instructions
13596
13597 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13598 match(Set dst (OrI src1 src2));
13599
13600 format %{ "orrw $dst, $src1, $src2\t# int" %}
13601
13602 ins_cost(INSN_COST);
13603 ins_encode %{
13604 __ orrw(as_Register($dst$$reg),
13605 as_Register($src1$$reg),
13606 as_Register($src2$$reg));
13607 %}
13608
13609 ins_pipe(ialu_reg_reg);
13610 %}
13611
13612 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13613 match(Set dst (OrI src1 src2));
13614
13615 format %{ "orrw $dst, $src1, $src2\t# int" %}
13616
13617 ins_cost(INSN_COST);
13618 ins_encode %{
13619 __ orrw(as_Register($dst$$reg),
13620 as_Register($src1$$reg),
13621 (uint64_t)($src2$$constant));
13622 %}
13623
13624 ins_pipe(ialu_reg_imm);
13625 %}
13626
13627 // Xor Instructions
13628
13629 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13630 match(Set dst (XorI src1 src2));
13631
13632 format %{ "eorw $dst, $src1, $src2\t# int" %}
13633
13634 ins_cost(INSN_COST);
13635 ins_encode %{
13636 __ eorw(as_Register($dst$$reg),
13637 as_Register($src1$$reg),
13638 as_Register($src2$$reg));
13639 %}
13640
13641 ins_pipe(ialu_reg_reg);
13642 %}
13643
13644 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13645 match(Set dst (XorI src1 src2));
13646
13647 format %{ "eorw $dst, $src1, $src2\t# int" %}
13648
13649 ins_cost(INSN_COST);
13650 ins_encode %{
13651 __ eorw(as_Register($dst$$reg),
13652 as_Register($src1$$reg),
13653 (uint64_t)($src2$$constant));
13654 %}
13655
13656 ins_pipe(ialu_reg_imm);
13657 %}
13658
13659 // Long Logical Instructions
13660 // TODO
13661
13662 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13663 match(Set dst (AndL src1 src2));
13664
13665 format %{ "and $dst, $src1, $src2\t# int" %}
13666
13667 ins_cost(INSN_COST);
13668 ins_encode %{
13669 __ andr(as_Register($dst$$reg),
13670 as_Register($src1$$reg),
13671 as_Register($src2$$reg));
13672 %}
13673
13674 ins_pipe(ialu_reg_reg);
13675 %}
13676
13677 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13678 match(Set dst (AndL src1 src2));
13679
13680 format %{ "and $dst, $src1, $src2\t# int" %}
13681
13682 ins_cost(INSN_COST);
13683 ins_encode %{
13684 __ andr(as_Register($dst$$reg),
13685 as_Register($src1$$reg),
13686 (uint64_t)($src2$$constant));
13687 %}
13688
13689 ins_pipe(ialu_reg_imm);
13690 %}
13691
13692 // Or Instructions
13693
13694 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13695 match(Set dst (OrL src1 src2));
13696
13697 format %{ "orr $dst, $src1, $src2\t# int" %}
13698
13699 ins_cost(INSN_COST);
13700 ins_encode %{
13701 __ orr(as_Register($dst$$reg),
13702 as_Register($src1$$reg),
13703 as_Register($src2$$reg));
13704 %}
13705
13706 ins_pipe(ialu_reg_reg);
13707 %}
13708
13709 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13710 match(Set dst (OrL src1 src2));
13711
13712 format %{ "orr $dst, $src1, $src2\t# int" %}
13713
13714 ins_cost(INSN_COST);
13715 ins_encode %{
13716 __ orr(as_Register($dst$$reg),
13717 as_Register($src1$$reg),
13718 (uint64_t)($src2$$constant));
13719 %}
13720
13721 ins_pipe(ialu_reg_imm);
13722 %}
13723
13724 // Xor Instructions
13725
13726 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13727 match(Set dst (XorL src1 src2));
13728
13729 format %{ "eor $dst, $src1, $src2\t# int" %}
13730
13731 ins_cost(INSN_COST);
13732 ins_encode %{
13733 __ eor(as_Register($dst$$reg),
13734 as_Register($src1$$reg),
13735 as_Register($src2$$reg));
13736 %}
13737
13738 ins_pipe(ialu_reg_reg);
13739 %}
13740
13741 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13742 match(Set dst (XorL src1 src2));
13743
13744 ins_cost(INSN_COST);
13745 format %{ "eor $dst, $src1, $src2\t# int" %}
13746
13747 ins_encode %{
13748 __ eor(as_Register($dst$$reg),
13749 as_Register($src1$$reg),
13750 (uint64_t)($src2$$constant));
13751 %}
13752
13753 ins_pipe(ialu_reg_imm);
13754 %}
13755
13756 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13757 %{
13758 match(Set dst (ConvI2L src));
13759
13760 ins_cost(INSN_COST);
13761 format %{ "sxtw $dst, $src\t# i2l" %}
13762 ins_encode %{
13763 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13764 %}
13765 ins_pipe(ialu_reg_shift);
13766 %}
13767
13768 // this pattern occurs in bigmath arithmetic
13769 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13770 %{
13771 match(Set dst (AndL (ConvI2L src) mask));
13772
13773 ins_cost(INSN_COST);
13774 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13775 ins_encode %{
13776 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13777 %}
13778
13779 ins_pipe(ialu_reg_shift);
13780 %}
13781
13782 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13783 match(Set dst (ConvL2I src));
13784
13785 ins_cost(INSN_COST);
13786 format %{ "movw $dst, $src \t// l2i" %}
13787
13788 ins_encode %{
13789 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13790 %}
13791
13792 ins_pipe(ialu_reg);
13793 %}
13794
13795 instruct convD2F_reg(vRegF dst, vRegD src) %{
13796 match(Set dst (ConvD2F src));
13797
13798 ins_cost(INSN_COST * 5);
13799 format %{ "fcvtd $dst, $src \t// d2f" %}
13800
13801 ins_encode %{
13802 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13803 %}
13804
13805 ins_pipe(fp_d2f);
13806 %}
13807
13808 instruct convF2D_reg(vRegD dst, vRegF src) %{
13809 match(Set dst (ConvF2D src));
13810
13811 ins_cost(INSN_COST * 5);
13812 format %{ "fcvts $dst, $src \t// f2d" %}
13813
13814 ins_encode %{
13815 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13816 %}
13817
13818 ins_pipe(fp_f2d);
13819 %}
13820
13821 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13822 match(Set dst (ConvF2I src));
13823
13824 ins_cost(INSN_COST * 5);
13825 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13826
13827 ins_encode %{
13828 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13829 %}
13830
13831 ins_pipe(fp_f2i);
13832 %}
13833
13834 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13835 match(Set dst (ConvF2L src));
13836
13837 ins_cost(INSN_COST * 5);
13838 format %{ "fcvtzs $dst, $src \t// f2l" %}
13839
13840 ins_encode %{
13841 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13842 %}
13843
13844 ins_pipe(fp_f2l);
13845 %}
13846
13847 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13848 match(Set dst (ConvF2HF src));
13849 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13850 "smov $dst, $tmp\t# move result from $tmp to $dst"
13851 %}
13852 effect(TEMP tmp);
13853 ins_encode %{
13854 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13855 %}
13856 ins_pipe(pipe_slow);
13857 %}
13858
13859 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13860 match(Set dst (ConvHF2F src));
13861 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13862 "fcvt $dst, $tmp\t# convert half to single precision"
13863 %}
13864 effect(TEMP tmp);
13865 ins_encode %{
13866 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13867 %}
13868 ins_pipe(pipe_slow);
13869 %}
13870
13871 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13872 match(Set dst (ConvI2F src));
13873
13874 ins_cost(INSN_COST * 5);
13875 format %{ "scvtfws $dst, $src \t// i2f" %}
13876
13877 ins_encode %{
13878 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13879 %}
13880
13881 ins_pipe(fp_i2f);
13882 %}
13883
13884 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13885 match(Set dst (ConvL2F src));
13886
13887 ins_cost(INSN_COST * 5);
13888 format %{ "scvtfs $dst, $src \t// l2f" %}
13889
13890 ins_encode %{
13891 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13892 %}
13893
13894 ins_pipe(fp_l2f);
13895 %}
13896
13897 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13898 match(Set dst (ConvD2I src));
13899
13900 ins_cost(INSN_COST * 5);
13901 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13902
13903 ins_encode %{
13904 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13905 %}
13906
13907 ins_pipe(fp_d2i);
13908 %}
13909
13910 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13911 match(Set dst (ConvD2L src));
13912
13913 ins_cost(INSN_COST * 5);
13914 format %{ "fcvtzd $dst, $src \t// d2l" %}
13915
13916 ins_encode %{
13917 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13918 %}
13919
13920 ins_pipe(fp_d2l);
13921 %}
13922
13923 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13924 match(Set dst (ConvI2D src));
13925
13926 ins_cost(INSN_COST * 5);
13927 format %{ "scvtfwd $dst, $src \t// i2d" %}
13928
13929 ins_encode %{
13930 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13931 %}
13932
13933 ins_pipe(fp_i2d);
13934 %}
13935
13936 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13937 match(Set dst (ConvL2D src));
13938
13939 ins_cost(INSN_COST * 5);
13940 format %{ "scvtfd $dst, $src \t// l2d" %}
13941
13942 ins_encode %{
13943 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13944 %}
13945
13946 ins_pipe(fp_l2d);
13947 %}
13948
13949 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13950 %{
13951 match(Set dst (RoundD src));
13952 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13953 format %{ "java_round_double $dst,$src"%}
13954 ins_encode %{
13955 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13956 as_FloatRegister($ftmp$$reg));
13957 %}
13958 ins_pipe(pipe_slow);
13959 %}
13960
13961 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13962 %{
13963 match(Set dst (RoundF src));
13964 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13965 format %{ "java_round_float $dst,$src"%}
13966 ins_encode %{
13967 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13968 as_FloatRegister($ftmp$$reg));
13969 %}
13970 ins_pipe(pipe_slow);
13971 %}
13972
13973 // stack <-> reg and reg <-> reg shuffles with no conversion
13974
13975 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13976
13977 match(Set dst (MoveF2I src));
13978
13979 effect(DEF dst, USE src);
13980
13981 ins_cost(4 * INSN_COST);
13982
13983 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13984
13985 ins_encode %{
13986 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13987 %}
13988
13989 ins_pipe(iload_reg_reg);
13990
13991 %}
13992
13993 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13994
13995 match(Set dst (MoveI2F src));
13996
13997 effect(DEF dst, USE src);
13998
13999 ins_cost(4 * INSN_COST);
14000
14001 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14002
14003 ins_encode %{
14004 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14005 %}
14006
14007 ins_pipe(pipe_class_memory);
14008
14009 %}
14010
14011 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14012
14013 match(Set dst (MoveD2L src));
14014
14015 effect(DEF dst, USE src);
14016
14017 ins_cost(4 * INSN_COST);
14018
14019 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14020
14021 ins_encode %{
14022 __ ldr($dst$$Register, Address(sp, $src$$disp));
14023 %}
14024
14025 ins_pipe(iload_reg_reg);
14026
14027 %}
14028
14029 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14030
14031 match(Set dst (MoveL2D src));
14032
14033 effect(DEF dst, USE src);
14034
14035 ins_cost(4 * INSN_COST);
14036
14037 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14038
14039 ins_encode %{
14040 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14041 %}
14042
14043 ins_pipe(pipe_class_memory);
14044
14045 %}
14046
14047 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14048
14049 match(Set dst (MoveF2I src));
14050
14051 effect(DEF dst, USE src);
14052
14053 ins_cost(INSN_COST);
14054
14055 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14056
14057 ins_encode %{
14058 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14059 %}
14060
14061 ins_pipe(pipe_class_memory);
14062
14063 %}
14064
14065 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14066
14067 match(Set dst (MoveI2F src));
14068
14069 effect(DEF dst, USE src);
14070
14071 ins_cost(INSN_COST);
14072
14073 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14074
14075 ins_encode %{
14076 __ strw($src$$Register, Address(sp, $dst$$disp));
14077 %}
14078
14079 ins_pipe(istore_reg_reg);
14080
14081 %}
14082
14083 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14084
14085 match(Set dst (MoveD2L src));
14086
14087 effect(DEF dst, USE src);
14088
14089 ins_cost(INSN_COST);
14090
14091 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14092
14093 ins_encode %{
14094 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14095 %}
14096
14097 ins_pipe(pipe_class_memory);
14098
14099 %}
14100
14101 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14102
14103 match(Set dst (MoveL2D src));
14104
14105 effect(DEF dst, USE src);
14106
14107 ins_cost(INSN_COST);
14108
14109 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14110
14111 ins_encode %{
14112 __ str($src$$Register, Address(sp, $dst$$disp));
14113 %}
14114
14115 ins_pipe(istore_reg_reg);
14116
14117 %}
14118
14119 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14120
14121 match(Set dst (MoveF2I src));
14122
14123 effect(DEF dst, USE src);
14124
14125 ins_cost(INSN_COST);
14126
14127 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14128
14129 ins_encode %{
14130 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14131 %}
14132
14133 ins_pipe(fp_f2i);
14134
14135 %}
14136
14137 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14138
14139 match(Set dst (MoveI2F src));
14140
14141 effect(DEF dst, USE src);
14142
14143 ins_cost(INSN_COST);
14144
14145 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14146
14147 ins_encode %{
14148 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14149 %}
14150
14151 ins_pipe(fp_i2f);
14152
14153 %}
14154
14155 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14156
14157 match(Set dst (MoveD2L src));
14158
14159 effect(DEF dst, USE src);
14160
14161 ins_cost(INSN_COST);
14162
14163 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14164
14165 ins_encode %{
14166 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14167 %}
14168
14169 ins_pipe(fp_d2l);
14170
14171 %}
14172
14173 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14174
14175 match(Set dst (MoveL2D src));
14176
14177 effect(DEF dst, USE src);
14178
14179 ins_cost(INSN_COST);
14180
14181 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14182
14183 ins_encode %{
14184 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14185 %}
14186
14187 ins_pipe(fp_l2d);
14188
14189 %}
14190
14191 // ============================================================================
14192 // clearing of an array
14193
14194 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14195 %{
14196 match(Set dummy (ClearArray (Binary cnt base) zero));
14197 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14198
14199 ins_cost(4 * INSN_COST);
14200 format %{ "ClearArray $cnt, $base" %}
14201
14202 ins_encode %{
14203 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14204 if (tpc == nullptr) {
14205 ciEnv::current()->record_failure("CodeCache is full");
14206 return;
14207 }
14208 %}
14209
14210 ins_pipe(pipe_class_memory);
14211 %}
14212
14213 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14214 %{
14215 predicate(((ClearArrayNode*)n)->word_copy_only());
14216 match(Set dummy (ClearArray (Binary cnt base) val));
14217 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14218
14219 ins_cost(4 * INSN_COST);
14220 format %{ "ClearArray $cnt, $base, $val" %}
14221
14222 ins_encode %{
14223 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14224 %}
14225
14226 ins_pipe(pipe_class_memory);
14227 %}
14228
14229 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, immL0 zero, Universe dummy, rFlagsReg cr)
14230 %{
14231 predicate((uint64_t)n->in(2)->in(1)->get_long()
14232 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14233 && !((ClearArrayNode*)n)->word_copy_only());
14234 match(Set dummy (ClearArray (Binary cnt base) zero));
14235 effect(TEMP temp, USE_KILL base, KILL cr);
14236
14237 ins_cost(4 * INSN_COST);
14238 format %{ "ClearArray $cnt, $base" %}
14239
14240 ins_encode %{
14241 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14242 if (tpc == nullptr) {
14243 ciEnv::current()->record_failure("CodeCache is full");
14244 return;
14245 }
14246 %}
14247
14248 ins_pipe(pipe_class_memory);
14249 %}
14250
14251 // ============================================================================
14252 // Overflow Math Instructions
14253
14254 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14255 %{
14256 match(Set cr (OverflowAddI op1 op2));
14257
14258 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14259 ins_cost(INSN_COST);
14260 ins_encode %{
14261 __ cmnw($op1$$Register, $op2$$Register);
14262 %}
14263
14264 ins_pipe(icmp_reg_reg);
14265 %}
14266
14267 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14268 %{
14269 match(Set cr (OverflowAddI op1 op2));
14270
14271 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14272 ins_cost(INSN_COST);
14273 ins_encode %{
14274 __ cmnw($op1$$Register, $op2$$constant);
14275 %}
14276
14277 ins_pipe(icmp_reg_imm);
14278 %}
14279
14280 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14281 %{
14282 match(Set cr (OverflowAddL op1 op2));
14283
14284 format %{ "cmn $op1, $op2\t# overflow check long" %}
14285 ins_cost(INSN_COST);
14286 ins_encode %{
14287 __ cmn($op1$$Register, $op2$$Register);
14288 %}
14289
14290 ins_pipe(icmp_reg_reg);
14291 %}
14292
14293 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14294 %{
14295 match(Set cr (OverflowAddL op1 op2));
14296
14297 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14298 ins_cost(INSN_COST);
14299 ins_encode %{
14300 __ adds(zr, $op1$$Register, $op2$$constant);
14301 %}
14302
14303 ins_pipe(icmp_reg_imm);
14304 %}
14305
14306 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14307 %{
14308 match(Set cr (OverflowSubI op1 op2));
14309
14310 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14311 ins_cost(INSN_COST);
14312 ins_encode %{
14313 __ cmpw($op1$$Register, $op2$$Register);
14314 %}
14315
14316 ins_pipe(icmp_reg_reg);
14317 %}
14318
14319 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14320 %{
14321 match(Set cr (OverflowSubI op1 op2));
14322
14323 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14324 ins_cost(INSN_COST);
14325 ins_encode %{
14326 __ cmpw($op1$$Register, $op2$$constant);
14327 %}
14328
14329 ins_pipe(icmp_reg_imm);
14330 %}
14331
14332 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14333 %{
14334 match(Set cr (OverflowSubL op1 op2));
14335
14336 format %{ "cmp $op1, $op2\t# overflow check long" %}
14337 ins_cost(INSN_COST);
14338 ins_encode %{
14339 __ cmp($op1$$Register, $op2$$Register);
14340 %}
14341
14342 ins_pipe(icmp_reg_reg);
14343 %}
14344
14345 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14346 %{
14347 match(Set cr (OverflowSubL op1 op2));
14348
14349 format %{ "cmp $op1, $op2\t# overflow check long" %}
14350 ins_cost(INSN_COST);
14351 ins_encode %{
14352 __ subs(zr, $op1$$Register, $op2$$constant);
14353 %}
14354
14355 ins_pipe(icmp_reg_imm);
14356 %}
14357
14358 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14359 %{
14360 match(Set cr (OverflowSubI zero op1));
14361
14362 format %{ "cmpw zr, $op1\t# overflow check int" %}
14363 ins_cost(INSN_COST);
14364 ins_encode %{
14365 __ cmpw(zr, $op1$$Register);
14366 %}
14367
14368 ins_pipe(icmp_reg_imm);
14369 %}
14370
14371 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14372 %{
14373 match(Set cr (OverflowSubL zero op1));
14374
14375 format %{ "cmp zr, $op1\t# overflow check long" %}
14376 ins_cost(INSN_COST);
14377 ins_encode %{
14378 __ cmp(zr, $op1$$Register);
14379 %}
14380
14381 ins_pipe(icmp_reg_imm);
14382 %}
14383
14384 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14385 %{
14386 match(Set cr (OverflowMulI op1 op2));
14387
14388 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14389 "cmp rscratch1, rscratch1, sxtw\n\t"
14390 "movw rscratch1, #0x80000000\n\t"
14391 "cselw rscratch1, rscratch1, zr, NE\n\t"
14392 "cmpw rscratch1, #1" %}
14393 ins_cost(5 * INSN_COST);
14394 ins_encode %{
14395 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14396 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14397 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14398 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14399 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14400 %}
14401
14402 ins_pipe(pipe_slow);
14403 %}
14404
14405 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14406 %{
14407 match(If cmp (OverflowMulI op1 op2));
14408 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14409 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14410 effect(USE labl, KILL cr);
14411
14412 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14413 "cmp rscratch1, rscratch1, sxtw\n\t"
14414 "b$cmp $labl" %}
14415 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14416 ins_encode %{
14417 Label* L = $labl$$label;
14418 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14419 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14420 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14421 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14422 %}
14423
14424 ins_pipe(pipe_serial);
14425 %}
14426
14427 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14428 %{
14429 match(Set cr (OverflowMulL op1 op2));
14430
14431 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14432 "smulh rscratch2, $op1, $op2\n\t"
14433 "cmp rscratch2, rscratch1, ASR #63\n\t"
14434 "movw rscratch1, #0x80000000\n\t"
14435 "cselw rscratch1, rscratch1, zr, NE\n\t"
14436 "cmpw rscratch1, #1" %}
14437 ins_cost(6 * INSN_COST);
14438 ins_encode %{
14439 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14440 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14441 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14442 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14443 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14444 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14445 %}
14446
14447 ins_pipe(pipe_slow);
14448 %}
14449
14450 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14451 %{
14452 match(If cmp (OverflowMulL op1 op2));
14453 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14454 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14455 effect(USE labl, KILL cr);
14456
14457 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14458 "smulh rscratch2, $op1, $op2\n\t"
14459 "cmp rscratch2, rscratch1, ASR #63\n\t"
14460 "b$cmp $labl" %}
14461 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14462 ins_encode %{
14463 Label* L = $labl$$label;
14464 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14465 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14466 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14467 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14468 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14469 %}
14470
14471 ins_pipe(pipe_serial);
14472 %}
14473
14474 // ============================================================================
14475 // Compare Instructions
14476
14477 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14478 %{
14479 match(Set cr (CmpI op1 op2));
14480
14481 effect(DEF cr, USE op1, USE op2);
14482
14483 ins_cost(INSN_COST);
14484 format %{ "cmpw $op1, $op2" %}
14485
14486 ins_encode(aarch64_enc_cmpw(op1, op2));
14487
14488 ins_pipe(icmp_reg_reg);
14489 %}
14490
14491 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14492 %{
14493 match(Set cr (CmpI op1 zero));
14494
14495 effect(DEF cr, USE op1);
14496
14497 ins_cost(INSN_COST);
14498 format %{ "cmpw $op1, 0" %}
14499
14500 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14501
14502 ins_pipe(icmp_reg_imm);
14503 %}
14504
14505 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14506 %{
14507 match(Set cr (CmpI op1 op2));
14508
14509 effect(DEF cr, USE op1);
14510
14511 ins_cost(INSN_COST);
14512 format %{ "cmpw $op1, $op2" %}
14513
14514 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14515
14516 ins_pipe(icmp_reg_imm);
14517 %}
14518
14519 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14520 %{
14521 match(Set cr (CmpI op1 op2));
14522
14523 effect(DEF cr, USE op1);
14524
14525 ins_cost(INSN_COST * 2);
14526 format %{ "cmpw $op1, $op2" %}
14527
14528 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14529
14530 ins_pipe(icmp_reg_imm);
14531 %}
14532
14533 // Unsigned compare Instructions; really, same as signed compare
14534 // except it should only be used to feed an If or a CMovI which takes a
14535 // cmpOpU.
14536
14537 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14538 %{
14539 match(Set cr (CmpU op1 op2));
14540
14541 effect(DEF cr, USE op1, USE op2);
14542
14543 ins_cost(INSN_COST);
14544 format %{ "cmpw $op1, $op2\t# unsigned" %}
14545
14546 ins_encode(aarch64_enc_cmpw(op1, op2));
14547
14548 ins_pipe(icmp_reg_reg);
14549 %}
14550
14551 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14552 %{
14553 match(Set cr (CmpU op1 zero));
14554
14555 effect(DEF cr, USE op1);
14556
14557 ins_cost(INSN_COST);
14558 format %{ "cmpw $op1, #0\t# unsigned" %}
14559
14560 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14561
14562 ins_pipe(icmp_reg_imm);
14563 %}
14564
14565 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14566 %{
14567 match(Set cr (CmpU op1 op2));
14568
14569 effect(DEF cr, USE op1);
14570
14571 ins_cost(INSN_COST);
14572 format %{ "cmpw $op1, $op2\t# unsigned" %}
14573
14574 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14575
14576 ins_pipe(icmp_reg_imm);
14577 %}
14578
14579 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14580 %{
14581 match(Set cr (CmpU op1 op2));
14582
14583 effect(DEF cr, USE op1);
14584
14585 ins_cost(INSN_COST * 2);
14586 format %{ "cmpw $op1, $op2\t# unsigned" %}
14587
14588 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14589
14590 ins_pipe(icmp_reg_imm);
14591 %}
14592
14593 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14594 %{
14595 match(Set cr (CmpL op1 op2));
14596
14597 effect(DEF cr, USE op1, USE op2);
14598
14599 ins_cost(INSN_COST);
14600 format %{ "cmp $op1, $op2" %}
14601
14602 ins_encode(aarch64_enc_cmp(op1, op2));
14603
14604 ins_pipe(icmp_reg_reg);
14605 %}
14606
14607 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14608 %{
14609 match(Set cr (CmpL op1 zero));
14610
14611 effect(DEF cr, USE op1);
14612
14613 ins_cost(INSN_COST);
14614 format %{ "tst $op1" %}
14615
14616 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14617
14618 ins_pipe(icmp_reg_imm);
14619 %}
14620
14621 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14622 %{
14623 match(Set cr (CmpL op1 op2));
14624
14625 effect(DEF cr, USE op1);
14626
14627 ins_cost(INSN_COST);
14628 format %{ "cmp $op1, $op2" %}
14629
14630 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14631
14632 ins_pipe(icmp_reg_imm);
14633 %}
14634
14635 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14636 %{
14637 match(Set cr (CmpL op1 op2));
14638
14639 effect(DEF cr, USE op1);
14640
14641 ins_cost(INSN_COST * 2);
14642 format %{ "cmp $op1, $op2" %}
14643
14644 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14645
14646 ins_pipe(icmp_reg_imm);
14647 %}
14648
14649 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14650 %{
14651 match(Set cr (CmpUL op1 op2));
14652
14653 effect(DEF cr, USE op1, USE op2);
14654
14655 ins_cost(INSN_COST);
14656 format %{ "cmp $op1, $op2" %}
14657
14658 ins_encode(aarch64_enc_cmp(op1, op2));
14659
14660 ins_pipe(icmp_reg_reg);
14661 %}
14662
14663 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14664 %{
14665 match(Set cr (CmpUL op1 zero));
14666
14667 effect(DEF cr, USE op1);
14668
14669 ins_cost(INSN_COST);
14670 format %{ "tst $op1" %}
14671
14672 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14673
14674 ins_pipe(icmp_reg_imm);
14675 %}
14676
14677 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14678 %{
14679 match(Set cr (CmpUL op1 op2));
14680
14681 effect(DEF cr, USE op1);
14682
14683 ins_cost(INSN_COST);
14684 format %{ "cmp $op1, $op2" %}
14685
14686 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14687
14688 ins_pipe(icmp_reg_imm);
14689 %}
14690
14691 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14692 %{
14693 match(Set cr (CmpUL op1 op2));
14694
14695 effect(DEF cr, USE op1);
14696
14697 ins_cost(INSN_COST * 2);
14698 format %{ "cmp $op1, $op2" %}
14699
14700 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14701
14702 ins_pipe(icmp_reg_imm);
14703 %}
14704
14705 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14706 %{
14707 match(Set cr (CmpP op1 op2));
14708
14709 effect(DEF cr, USE op1, USE op2);
14710
14711 ins_cost(INSN_COST);
14712 format %{ "cmp $op1, $op2\t // ptr" %}
14713
14714 ins_encode(aarch64_enc_cmpp(op1, op2));
14715
14716 ins_pipe(icmp_reg_reg);
14717 %}
14718
14719 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14720 %{
14721 match(Set cr (CmpN op1 op2));
14722
14723 effect(DEF cr, USE op1, USE op2);
14724
14725 ins_cost(INSN_COST);
14726 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14727
14728 ins_encode(aarch64_enc_cmpn(op1, op2));
14729
14730 ins_pipe(icmp_reg_reg);
14731 %}
14732
14733 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14734 %{
14735 match(Set cr (CmpP op1 zero));
14736
14737 effect(DEF cr, USE op1, USE zero);
14738
14739 ins_cost(INSN_COST);
14740 format %{ "cmp $op1, 0\t // ptr" %}
14741
14742 ins_encode(aarch64_enc_testp(op1));
14743
14744 ins_pipe(icmp_reg_imm);
14745 %}
14746
14747 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14748 %{
14749 match(Set cr (CmpN op1 zero));
14750
14751 effect(DEF cr, USE op1, USE zero);
14752
14753 ins_cost(INSN_COST);
14754 format %{ "cmp $op1, 0\t // compressed ptr" %}
14755
14756 ins_encode(aarch64_enc_testn(op1));
14757
14758 ins_pipe(icmp_reg_imm);
14759 %}
14760
14761 // FP comparisons
14762 //
14763 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14764 // using normal cmpOp. See declaration of rFlagsReg for details.
14765
14766 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14767 %{
14768 match(Set cr (CmpF src1 src2));
14769
14770 ins_cost(3 * INSN_COST);
14771 format %{ "fcmps $src1, $src2" %}
14772
14773 ins_encode %{
14774 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14775 %}
14776
14777 ins_pipe(pipe_class_compare);
14778 %}
14779
14780 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14781 %{
14782 match(Set cr (CmpF src1 src2));
14783
14784 ins_cost(3 * INSN_COST);
14785 format %{ "fcmps $src1, 0.0" %}
14786
14787 ins_encode %{
14788 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14789 %}
14790
14791 ins_pipe(pipe_class_compare);
14792 %}
14793 // FROM HERE
14794
14795 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14796 %{
14797 match(Set cr (CmpD src1 src2));
14798
14799 ins_cost(3 * INSN_COST);
14800 format %{ "fcmpd $src1, $src2" %}
14801
14802 ins_encode %{
14803 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14804 %}
14805
14806 ins_pipe(pipe_class_compare);
14807 %}
14808
14809 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14810 %{
14811 match(Set cr (CmpD src1 src2));
14812
14813 ins_cost(3 * INSN_COST);
14814 format %{ "fcmpd $src1, 0.0" %}
14815
14816 ins_encode %{
14817 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14818 %}
14819
14820 ins_pipe(pipe_class_compare);
14821 %}
14822
14823 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14824 %{
14825 match(Set dst (CmpF3 src1 src2));
14826 effect(KILL cr);
14827
14828 ins_cost(5 * INSN_COST);
14829 format %{ "fcmps $src1, $src2\n\t"
14830 "csinvw($dst, zr, zr, eq\n\t"
14831 "csnegw($dst, $dst, $dst, lt)"
14832 %}
14833
14834 ins_encode %{
14835 Label done;
14836 FloatRegister s1 = as_FloatRegister($src1$$reg);
14837 FloatRegister s2 = as_FloatRegister($src2$$reg);
14838 Register d = as_Register($dst$$reg);
14839 __ fcmps(s1, s2);
14840 // installs 0 if EQ else -1
14841 __ csinvw(d, zr, zr, Assembler::EQ);
14842 // keeps -1 if less or unordered else installs 1
14843 __ csnegw(d, d, d, Assembler::LT);
14844 __ bind(done);
14845 %}
14846
14847 ins_pipe(pipe_class_default);
14848
14849 %}
14850
14851 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14852 %{
14853 match(Set dst (CmpD3 src1 src2));
14854 effect(KILL cr);
14855
14856 ins_cost(5 * INSN_COST);
14857 format %{ "fcmpd $src1, $src2\n\t"
14858 "csinvw($dst, zr, zr, eq\n\t"
14859 "csnegw($dst, $dst, $dst, lt)"
14860 %}
14861
14862 ins_encode %{
14863 Label done;
14864 FloatRegister s1 = as_FloatRegister($src1$$reg);
14865 FloatRegister s2 = as_FloatRegister($src2$$reg);
14866 Register d = as_Register($dst$$reg);
14867 __ fcmpd(s1, s2);
14868 // installs 0 if EQ else -1
14869 __ csinvw(d, zr, zr, Assembler::EQ);
14870 // keeps -1 if less or unordered else installs 1
14871 __ csnegw(d, d, d, Assembler::LT);
14872 __ bind(done);
14873 %}
14874 ins_pipe(pipe_class_default);
14875
14876 %}
14877
14878 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14879 %{
14880 match(Set dst (CmpF3 src1 zero));
14881 effect(KILL cr);
14882
14883 ins_cost(5 * INSN_COST);
14884 format %{ "fcmps $src1, 0.0\n\t"
14885 "csinvw($dst, zr, zr, eq\n\t"
14886 "csnegw($dst, $dst, $dst, lt)"
14887 %}
14888
14889 ins_encode %{
14890 Label done;
14891 FloatRegister s1 = as_FloatRegister($src1$$reg);
14892 Register d = as_Register($dst$$reg);
14893 __ fcmps(s1, 0.0);
14894 // installs 0 if EQ else -1
14895 __ csinvw(d, zr, zr, Assembler::EQ);
14896 // keeps -1 if less or unordered else installs 1
14897 __ csnegw(d, d, d, Assembler::LT);
14898 __ bind(done);
14899 %}
14900
14901 ins_pipe(pipe_class_default);
14902
14903 %}
14904
14905 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14906 %{
14907 match(Set dst (CmpD3 src1 zero));
14908 effect(KILL cr);
14909
14910 ins_cost(5 * INSN_COST);
14911 format %{ "fcmpd $src1, 0.0\n\t"
14912 "csinvw($dst, zr, zr, eq\n\t"
14913 "csnegw($dst, $dst, $dst, lt)"
14914 %}
14915
14916 ins_encode %{
14917 Label done;
14918 FloatRegister s1 = as_FloatRegister($src1$$reg);
14919 Register d = as_Register($dst$$reg);
14920 __ fcmpd(s1, 0.0);
14921 // installs 0 if EQ else -1
14922 __ csinvw(d, zr, zr, Assembler::EQ);
14923 // keeps -1 if less or unordered else installs 1
14924 __ csnegw(d, d, d, Assembler::LT);
14925 __ bind(done);
14926 %}
14927 ins_pipe(pipe_class_default);
14928
14929 %}
14930
14931 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14932 %{
14933 match(Set dst (CmpLTMask p q));
14934 effect(KILL cr);
14935
14936 ins_cost(3 * INSN_COST);
14937
14938 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14939 "csetw $dst, lt\n\t"
14940 "subw $dst, zr, $dst"
14941 %}
14942
14943 ins_encode %{
14944 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14945 __ csetw(as_Register($dst$$reg), Assembler::LT);
14946 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14947 %}
14948
14949 ins_pipe(ialu_reg_reg);
14950 %}
14951
14952 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14953 %{
14954 match(Set dst (CmpLTMask src zero));
14955 effect(KILL cr);
14956
14957 ins_cost(INSN_COST);
14958
14959 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14960
14961 ins_encode %{
14962 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14963 %}
14964
14965 ins_pipe(ialu_reg_shift);
14966 %}
14967
14968 // ============================================================================
14969 // Max and Min
14970
14971 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14972
14973 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14974 %{
14975 effect(DEF cr, USE src);
14976 ins_cost(INSN_COST);
14977 format %{ "cmpw $src, 0" %}
14978
14979 ins_encode %{
14980 __ cmpw($src$$Register, 0);
14981 %}
14982 ins_pipe(icmp_reg_imm);
14983 %}
14984
14985 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14986 %{
14987 match(Set dst (MinI src1 src2));
14988 ins_cost(INSN_COST * 3);
14989
14990 expand %{
14991 rFlagsReg cr;
14992 compI_reg_reg(cr, src1, src2);
14993 cmovI_reg_reg_lt(dst, src1, src2, cr);
14994 %}
14995 %}
14996
14997 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14998 %{
14999 match(Set dst (MaxI src1 src2));
15000 ins_cost(INSN_COST * 3);
15001
15002 expand %{
15003 rFlagsReg cr;
15004 compI_reg_reg(cr, src1, src2);
15005 cmovI_reg_reg_gt(dst, src1, src2, cr);
15006 %}
15007 %}
15008
15009
15010 // ============================================================================
15011 // Branch Instructions
15012
15013 // Direct Branch.
15014 instruct branch(label lbl)
15015 %{
15016 match(Goto);
15017
15018 effect(USE lbl);
15019
15020 ins_cost(BRANCH_COST);
15021 format %{ "b $lbl" %}
15022
15023 ins_encode(aarch64_enc_b(lbl));
15024
15025 ins_pipe(pipe_branch);
15026 %}
15027
15028 // Conditional Near Branch
15029 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15030 %{
15031 // Same match rule as `branchConFar'.
15032 match(If cmp cr);
15033
15034 effect(USE lbl);
15035
15036 ins_cost(BRANCH_COST);
15037 // If set to 1 this indicates that the current instruction is a
15038 // short variant of a long branch. This avoids using this
15039 // instruction in first-pass matching. It will then only be used in
15040 // the `Shorten_branches' pass.
15041 // ins_short_branch(1);
15042 format %{ "b$cmp $lbl" %}
15043
15044 ins_encode(aarch64_enc_br_con(cmp, lbl));
15045
15046 ins_pipe(pipe_branch_cond);
15047 %}
15048
15049 // Conditional Near Branch Unsigned
15050 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15051 %{
15052 // Same match rule as `branchConFar'.
15053 match(If cmp cr);
15054
15055 effect(USE lbl);
15056
15057 ins_cost(BRANCH_COST);
15058 // If set to 1 this indicates that the current instruction is a
15059 // short variant of a long branch. This avoids using this
15060 // instruction in first-pass matching. It will then only be used in
15061 // the `Shorten_branches' pass.
15062 // ins_short_branch(1);
15063 format %{ "b$cmp $lbl\t# unsigned" %}
15064
15065 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15066
15067 ins_pipe(pipe_branch_cond);
15068 %}
15069
15070 // Make use of CBZ and CBNZ. These instructions, as well as being
15071 // shorter than (cmp; branch), have the additional benefit of not
15072 // killing the flags.
15073
15074 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15075 match(If cmp (CmpI op1 op2));
15076 effect(USE labl);
15077
15078 ins_cost(BRANCH_COST);
15079 format %{ "cbw$cmp $op1, $labl" %}
15080 ins_encode %{
15081 Label* L = $labl$$label;
15082 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15083 if (cond == Assembler::EQ)
15084 __ cbzw($op1$$Register, *L);
15085 else
15086 __ cbnzw($op1$$Register, *L);
15087 %}
15088 ins_pipe(pipe_cmp_branch);
15089 %}
15090
15091 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15092 match(If cmp (CmpL op1 op2));
15093 effect(USE labl);
15094
15095 ins_cost(BRANCH_COST);
15096 format %{ "cb$cmp $op1, $labl" %}
15097 ins_encode %{
15098 Label* L = $labl$$label;
15099 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15100 if (cond == Assembler::EQ)
15101 __ cbz($op1$$Register, *L);
15102 else
15103 __ cbnz($op1$$Register, *L);
15104 %}
15105 ins_pipe(pipe_cmp_branch);
15106 %}
15107
15108 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15109 match(If cmp (CmpP op1 op2));
15110 effect(USE labl);
15111
15112 ins_cost(BRANCH_COST);
15113 format %{ "cb$cmp $op1, $labl" %}
15114 ins_encode %{
15115 Label* L = $labl$$label;
15116 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15117 if (cond == Assembler::EQ)
15118 __ cbz($op1$$Register, *L);
15119 else
15120 __ cbnz($op1$$Register, *L);
15121 %}
15122 ins_pipe(pipe_cmp_branch);
15123 %}
15124
15125 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15126 match(If cmp (CmpN op1 op2));
15127 effect(USE labl);
15128
15129 ins_cost(BRANCH_COST);
15130 format %{ "cbw$cmp $op1, $labl" %}
15131 ins_encode %{
15132 Label* L = $labl$$label;
15133 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15134 if (cond == Assembler::EQ)
15135 __ cbzw($op1$$Register, *L);
15136 else
15137 __ cbnzw($op1$$Register, *L);
15138 %}
15139 ins_pipe(pipe_cmp_branch);
15140 %}
15141
15142 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15143 match(If cmp (CmpP (DecodeN oop) zero));
15144 effect(USE labl);
15145
15146 ins_cost(BRANCH_COST);
15147 format %{ "cb$cmp $oop, $labl" %}
15148 ins_encode %{
15149 Label* L = $labl$$label;
15150 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15151 if (cond == Assembler::EQ)
15152 __ cbzw($oop$$Register, *L);
15153 else
15154 __ cbnzw($oop$$Register, *L);
15155 %}
15156 ins_pipe(pipe_cmp_branch);
15157 %}
15158
15159 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15160 match(If cmp (CmpU op1 op2));
15161 effect(USE labl);
15162
15163 ins_cost(BRANCH_COST);
15164 format %{ "cbw$cmp $op1, $labl" %}
15165 ins_encode %{
15166 Label* L = $labl$$label;
15167 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15168 if (cond == Assembler::EQ || cond == Assembler::LS) {
15169 __ cbzw($op1$$Register, *L);
15170 } else {
15171 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15172 __ cbnzw($op1$$Register, *L);
15173 }
15174 %}
15175 ins_pipe(pipe_cmp_branch);
15176 %}
15177
15178 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15179 match(If cmp (CmpUL op1 op2));
15180 effect(USE labl);
15181
15182 ins_cost(BRANCH_COST);
15183 format %{ "cb$cmp $op1, $labl" %}
15184 ins_encode %{
15185 Label* L = $labl$$label;
15186 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15187 if (cond == Assembler::EQ || cond == Assembler::LS) {
15188 __ cbz($op1$$Register, *L);
15189 } else {
15190 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15191 __ cbnz($op1$$Register, *L);
15192 }
15193 %}
15194 ins_pipe(pipe_cmp_branch);
15195 %}
15196
15197 // Test bit and Branch
15198
15199 // Patterns for short (< 32KiB) variants
15200 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15201 match(If cmp (CmpL op1 op2));
15202 effect(USE labl);
15203
15204 ins_cost(BRANCH_COST);
15205 format %{ "cb$cmp $op1, $labl # long" %}
15206 ins_encode %{
15207 Label* L = $labl$$label;
15208 Assembler::Condition cond =
15209 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15210 __ tbr(cond, $op1$$Register, 63, *L);
15211 %}
15212 ins_pipe(pipe_cmp_branch);
15213 ins_short_branch(1);
15214 %}
15215
15216 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15217 match(If cmp (CmpI op1 op2));
15218 effect(USE labl);
15219
15220 ins_cost(BRANCH_COST);
15221 format %{ "cb$cmp $op1, $labl # int" %}
15222 ins_encode %{
15223 Label* L = $labl$$label;
15224 Assembler::Condition cond =
15225 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15226 __ tbr(cond, $op1$$Register, 31, *L);
15227 %}
15228 ins_pipe(pipe_cmp_branch);
15229 ins_short_branch(1);
15230 %}
15231
15232 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15233 match(If cmp (CmpL (AndL op1 op2) op3));
15234 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15235 effect(USE labl);
15236
15237 ins_cost(BRANCH_COST);
15238 format %{ "tb$cmp $op1, $op2, $labl" %}
15239 ins_encode %{
15240 Label* L = $labl$$label;
15241 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15242 int bit = exact_log2_long($op2$$constant);
15243 __ tbr(cond, $op1$$Register, bit, *L);
15244 %}
15245 ins_pipe(pipe_cmp_branch);
15246 ins_short_branch(1);
15247 %}
15248
15249 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15250 match(If cmp (CmpI (AndI op1 op2) op3));
15251 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15252 effect(USE labl);
15253
15254 ins_cost(BRANCH_COST);
15255 format %{ "tb$cmp $op1, $op2, $labl" %}
15256 ins_encode %{
15257 Label* L = $labl$$label;
15258 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15259 int bit = exact_log2((juint)$op2$$constant);
15260 __ tbr(cond, $op1$$Register, bit, *L);
15261 %}
15262 ins_pipe(pipe_cmp_branch);
15263 ins_short_branch(1);
15264 %}
15265
15266 // And far variants
15267 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15268 match(If cmp (CmpL op1 op2));
15269 effect(USE labl);
15270
15271 ins_cost(BRANCH_COST);
15272 format %{ "cb$cmp $op1, $labl # long" %}
15273 ins_encode %{
15274 Label* L = $labl$$label;
15275 Assembler::Condition cond =
15276 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15277 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15278 %}
15279 ins_pipe(pipe_cmp_branch);
15280 %}
15281
15282 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15283 match(If cmp (CmpI op1 op2));
15284 effect(USE labl);
15285
15286 ins_cost(BRANCH_COST);
15287 format %{ "cb$cmp $op1, $labl # int" %}
15288 ins_encode %{
15289 Label* L = $labl$$label;
15290 Assembler::Condition cond =
15291 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15292 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15293 %}
15294 ins_pipe(pipe_cmp_branch);
15295 %}
15296
15297 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15298 match(If cmp (CmpL (AndL op1 op2) op3));
15299 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15300 effect(USE labl);
15301
15302 ins_cost(BRANCH_COST);
15303 format %{ "tb$cmp $op1, $op2, $labl" %}
15304 ins_encode %{
15305 Label* L = $labl$$label;
15306 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15307 int bit = exact_log2_long($op2$$constant);
15308 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15309 %}
15310 ins_pipe(pipe_cmp_branch);
15311 %}
15312
15313 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15314 match(If cmp (CmpI (AndI op1 op2) op3));
15315 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15316 effect(USE labl);
15317
15318 ins_cost(BRANCH_COST);
15319 format %{ "tb$cmp $op1, $op2, $labl" %}
15320 ins_encode %{
15321 Label* L = $labl$$label;
15322 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15323 int bit = exact_log2((juint)$op2$$constant);
15324 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15325 %}
15326 ins_pipe(pipe_cmp_branch);
15327 %}
15328
15329 // Test bits
15330
15331 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15332 match(Set cr (CmpL (AndL op1 op2) op3));
15333 predicate(Assembler::operand_valid_for_logical_immediate
15334 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15335
15336 ins_cost(INSN_COST);
15337 format %{ "tst $op1, $op2 # long" %}
15338 ins_encode %{
15339 __ tst($op1$$Register, $op2$$constant);
15340 %}
15341 ins_pipe(ialu_reg_reg);
15342 %}
15343
15344 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15345 match(Set cr (CmpI (AndI op1 op2) op3));
15346 predicate(Assembler::operand_valid_for_logical_immediate
15347 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15348
15349 ins_cost(INSN_COST);
15350 format %{ "tst $op1, $op2 # int" %}
15351 ins_encode %{
15352 __ tstw($op1$$Register, $op2$$constant);
15353 %}
15354 ins_pipe(ialu_reg_reg);
15355 %}
15356
15357 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15358 match(Set cr (CmpL (AndL op1 op2) op3));
15359
15360 ins_cost(INSN_COST);
15361 format %{ "tst $op1, $op2 # long" %}
15362 ins_encode %{
15363 __ tst($op1$$Register, $op2$$Register);
15364 %}
15365 ins_pipe(ialu_reg_reg);
15366 %}
15367
15368 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15369 match(Set cr (CmpI (AndI op1 op2) op3));
15370
15371 ins_cost(INSN_COST);
15372 format %{ "tstw $op1, $op2 # int" %}
15373 ins_encode %{
15374 __ tstw($op1$$Register, $op2$$Register);
15375 %}
15376 ins_pipe(ialu_reg_reg);
15377 %}
15378
15379
15380 // Conditional Far Branch
15381 // Conditional Far Branch Unsigned
15382 // TODO: fixme
15383
15384 // counted loop end branch near
15385 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15386 %{
15387 match(CountedLoopEnd cmp cr);
15388
15389 effect(USE lbl);
15390
15391 ins_cost(BRANCH_COST);
15392 // short variant.
15393 // ins_short_branch(1);
15394 format %{ "b$cmp $lbl \t// counted loop end" %}
15395
15396 ins_encode(aarch64_enc_br_con(cmp, lbl));
15397
15398 ins_pipe(pipe_branch);
15399 %}
15400
15401 // counted loop end branch far
15402 // TODO: fixme
15403
15404 // ============================================================================
15405 // inlined locking and unlocking
15406
15407 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15408 %{
15409 match(Set cr (FastLock object box));
15410 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15411
15412 ins_cost(5 * INSN_COST);
15413 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15414
15415 ins_encode %{
15416 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15417 %}
15418
15419 ins_pipe(pipe_serial);
15420 %}
15421
15422 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15423 %{
15424 match(Set cr (FastUnlock object box));
15425 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15426
15427 ins_cost(5 * INSN_COST);
15428 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15429
15430 ins_encode %{
15431 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15432 %}
15433
15434 ins_pipe(pipe_serial);
15435 %}
15436
15437 // ============================================================================
15438 // Safepoint Instructions
15439
15440 // TODO
15441 // provide a near and far version of this code
15442
15443 instruct safePoint(rFlagsReg cr, iRegP poll)
15444 %{
15445 match(SafePoint poll);
15446 effect(KILL cr);
15447
15448 format %{
15449 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15450 %}
15451 ins_encode %{
15452 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15453 %}
15454 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15455 %}
15456
15457
15458 // ============================================================================
15459 // Procedure Call/Return Instructions
15460
15461 // Call Java Static Instruction
15462
15463 instruct CallStaticJavaDirect(method meth)
15464 %{
15465 match(CallStaticJava);
15466
15467 effect(USE meth);
15468
15469 ins_cost(CALL_COST);
15470
15471 format %{ "call,static $meth \t// ==> " %}
15472
15473 ins_encode(aarch64_enc_java_static_call(meth),
15474 aarch64_enc_call_epilog);
15475
15476 ins_pipe(pipe_class_call);
15477 %}
15478
15479 // TO HERE
15480
15481 // Call Java Dynamic Instruction
15482 instruct CallDynamicJavaDirect(method meth)
15483 %{
15484 match(CallDynamicJava);
15485
15486 effect(USE meth);
15487
15488 ins_cost(CALL_COST);
15489
15490 format %{ "CALL,dynamic $meth \t// ==> " %}
15491
15492 ins_encode(aarch64_enc_java_dynamic_call(meth),
15493 aarch64_enc_call_epilog);
15494
15495 ins_pipe(pipe_class_call);
15496 %}
15497
15498 // Call Runtime Instruction
15499
15500 instruct CallRuntimeDirect(method meth)
15501 %{
15502 match(CallRuntime);
15503
15504 effect(USE meth);
15505
15506 ins_cost(CALL_COST);
15507
15508 format %{ "CALL, runtime $meth" %}
15509
15510 ins_encode( aarch64_enc_java_to_runtime(meth) );
15511
15512 ins_pipe(pipe_class_call);
15513 %}
15514
15515 // Call Runtime Instruction
15516
15517 instruct CallLeafDirect(method meth)
15518 %{
15519 match(CallLeaf);
15520
15521 effect(USE meth);
15522
15523 ins_cost(CALL_COST);
15524
15525 format %{ "CALL, runtime leaf $meth" %}
15526
15527 ins_encode( aarch64_enc_java_to_runtime(meth) );
15528
15529 ins_pipe(pipe_class_call);
15530 %}
15531
15532 // Call Runtime Instruction without safepoint and with vector arguments
15533 instruct CallLeafDirectVector(method meth)
15534 %{
15535 match(CallLeafVector);
15536
15537 effect(USE meth);
15538
15539 ins_cost(CALL_COST);
15540
15541 format %{ "CALL, runtime leaf vector $meth" %}
15542
15543 ins_encode(aarch64_enc_java_to_runtime(meth));
15544
15545 ins_pipe(pipe_class_call);
15546 %}
15547
15548 // Call Runtime Instruction
15549
15550 // entry point is null, target holds the address to call
15551 instruct CallLeafNoFPIndirect(iRegP target)
15552 %{
15553 predicate(n->as_Call()->entry_point() == nullptr);
15554
15555 match(CallLeafNoFP target);
15556
15557 ins_cost(CALL_COST);
15558
15559 format %{ "CALL, runtime leaf nofp indirect $target" %}
15560
15561 ins_encode %{
15562 __ blr($target$$Register);
15563 %}
15564
15565 ins_pipe(pipe_class_call);
15566 %}
15567
15568 instruct CallLeafNoFPDirect(method meth)
15569 %{
15570 predicate(n->as_Call()->entry_point() != nullptr);
15571
15572 match(CallLeafNoFP);
15573
15574 effect(USE meth);
15575
15576 ins_cost(CALL_COST);
15577
15578 format %{ "CALL, runtime leaf nofp $meth" %}
15579
15580 ins_encode( aarch64_enc_java_to_runtime(meth) );
15581
15582 ins_pipe(pipe_class_call);
15583 %}
15584
15585 // Tail Call; Jump from runtime stub to Java code.
15586 // Also known as an 'interprocedural jump'.
15587 // Target of jump will eventually return to caller.
15588 // TailJump below removes the return address.
15589 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15590 // emitted just above the TailCall which has reset rfp to the caller state.
15591 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15592 %{
15593 match(TailCall jump_target method_ptr);
15594
15595 ins_cost(CALL_COST);
15596
15597 format %{ "br $jump_target\t# $method_ptr holds method" %}
15598
15599 ins_encode(aarch64_enc_tail_call(jump_target));
15600
15601 ins_pipe(pipe_class_call);
15602 %}
15603
15604 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15605 %{
15606 match(TailJump jump_target ex_oop);
15607
15608 ins_cost(CALL_COST);
15609
15610 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15611
15612 ins_encode(aarch64_enc_tail_jmp(jump_target));
15613
15614 ins_pipe(pipe_class_call);
15615 %}
15616
15617 // Forward exception.
15618 instruct ForwardExceptionjmp()
15619 %{
15620 match(ForwardException);
15621 ins_cost(CALL_COST);
15622
15623 format %{ "b forward_exception_stub" %}
15624 ins_encode %{
15625 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15626 %}
15627 ins_pipe(pipe_class_call);
15628 %}
15629
15630 // Create exception oop: created by stack-crawling runtime code.
15631 // Created exception is now available to this handler, and is setup
15632 // just prior to jumping to this handler. No code emitted.
15633 // TODO check
15634 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15635 instruct CreateException(iRegP_R0 ex_oop)
15636 %{
15637 match(Set ex_oop (CreateEx));
15638
15639 format %{ " -- \t// exception oop; no code emitted" %}
15640
15641 size(0);
15642
15643 ins_encode( /*empty*/ );
15644
15645 ins_pipe(pipe_class_empty);
15646 %}
15647
15648 // Rethrow exception: The exception oop will come in the first
15649 // argument position. Then JUMP (not call) to the rethrow stub code.
15650 instruct RethrowException() %{
15651 match(Rethrow);
15652 ins_cost(CALL_COST);
15653
15654 format %{ "b rethrow_stub" %}
15655
15656 ins_encode( aarch64_enc_rethrow() );
15657
15658 ins_pipe(pipe_class_call);
15659 %}
15660
15661
15662 // Return Instruction
15663 // epilog node loads ret address into lr as part of frame pop
15664 instruct Ret()
15665 %{
15666 match(Return);
15667
15668 format %{ "ret\t// return register" %}
15669
15670 ins_encode( aarch64_enc_ret() );
15671
15672 ins_pipe(pipe_branch);
15673 %}
15674
15675 // Die now.
15676 instruct ShouldNotReachHere() %{
15677 match(Halt);
15678
15679 ins_cost(CALL_COST);
15680 format %{ "ShouldNotReachHere" %}
15681
15682 ins_encode %{
15683 if (is_reachable()) {
15684 const char* str = __ code_string(_halt_reason);
15685 __ stop(str);
15686 }
15687 %}
15688
15689 ins_pipe(pipe_class_default);
15690 %}
15691
15692 // ============================================================================
15693 // Partial Subtype Check
15694 //
15695 // superklass array for an instance of the superklass. Set a hidden
15696 // internal cache on a hit (cache is checked with exposed code in
15697 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15698 // encoding ALSO sets flags.
15699
15700 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15701 %{
15702 match(Set result (PartialSubtypeCheck sub super));
15703 predicate(!UseSecondarySupersTable);
15704 effect(KILL cr, KILL temp);
15705
15706 ins_cost(20 * INSN_COST); // slightly larger than the next version
15707 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15708
15709 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15710
15711 opcode(0x1); // Force zero of result reg on hit
15712
15713 ins_pipe(pipe_class_memory);
15714 %}
15715
15716 // Two versions of partialSubtypeCheck, both used when we need to
15717 // search for a super class in the secondary supers array. The first
15718 // is used when we don't know _a priori_ the class being searched
15719 // for. The second, far more common, is used when we do know: this is
15720 // used for instanceof, checkcast, and any case where C2 can determine
15721 // it by constant propagation.
15722
15723 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15724 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15725 rFlagsReg cr)
15726 %{
15727 match(Set result (PartialSubtypeCheck sub super));
15728 predicate(UseSecondarySupersTable);
15729 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15730
15731 ins_cost(10 * INSN_COST); // slightly larger than the next version
15732 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15733
15734 ins_encode %{
15735 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15736 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15737 $vtemp$$FloatRegister,
15738 $result$$Register, /*L_success*/nullptr);
15739 %}
15740
15741 ins_pipe(pipe_class_memory);
15742 %}
15743
15744 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15745 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15746 rFlagsReg cr)
15747 %{
15748 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15749 predicate(UseSecondarySupersTable);
15750 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15751
15752 ins_cost(5 * INSN_COST); // smaller than the next version
15753 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15754
15755 ins_encode %{
15756 bool success = false;
15757 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15758 if (InlineSecondarySupersTest) {
15759 success =
15760 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15761 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15762 $vtemp$$FloatRegister,
15763 $result$$Register,
15764 super_klass_slot);
15765 } else {
15766 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15767 success = (call != nullptr);
15768 }
15769 if (!success) {
15770 ciEnv::current()->record_failure("CodeCache is full");
15771 return;
15772 }
15773 %}
15774
15775 ins_pipe(pipe_class_memory);
15776 %}
15777
15778 // Intrisics for String.compareTo()
15779
15780 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15781 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15782 %{
15783 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15784 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15785 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15786
15787 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15788 ins_encode %{
15789 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15790 __ string_compare($str1$$Register, $str2$$Register,
15791 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15792 $tmp1$$Register, $tmp2$$Register,
15793 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15794 %}
15795 ins_pipe(pipe_class_memory);
15796 %}
15797
15798 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15799 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15800 %{
15801 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15802 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15803 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15804
15805 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15806 ins_encode %{
15807 __ string_compare($str1$$Register, $str2$$Register,
15808 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15809 $tmp1$$Register, $tmp2$$Register,
15810 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15811 %}
15812 ins_pipe(pipe_class_memory);
15813 %}
15814
15815 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15816 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15817 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15818 %{
15819 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15820 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15821 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15822 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15823
15824 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15825 ins_encode %{
15826 __ string_compare($str1$$Register, $str2$$Register,
15827 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15828 $tmp1$$Register, $tmp2$$Register,
15829 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15830 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15831 %}
15832 ins_pipe(pipe_class_memory);
15833 %}
15834
15835 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15836 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15837 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15838 %{
15839 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15840 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15841 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15842 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15843
15844 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15845 ins_encode %{
15846 __ string_compare($str1$$Register, $str2$$Register,
15847 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15848 $tmp1$$Register, $tmp2$$Register,
15849 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15850 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15851 %}
15852 ins_pipe(pipe_class_memory);
15853 %}
15854
15855 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15856 // these string_compare variants as NEON register type for convenience so that the prototype of
15857 // string_compare can be shared with all variants.
15858
15859 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15860 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15861 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15862 pRegGov_P1 pgtmp2, rFlagsReg cr)
15863 %{
15864 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15865 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15866 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15867 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15868
15869 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15870 ins_encode %{
15871 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15872 __ string_compare($str1$$Register, $str2$$Register,
15873 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15874 $tmp1$$Register, $tmp2$$Register,
15875 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15876 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15877 StrIntrinsicNode::LL);
15878 %}
15879 ins_pipe(pipe_class_memory);
15880 %}
15881
15882 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15883 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15884 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15885 pRegGov_P1 pgtmp2, rFlagsReg cr)
15886 %{
15887 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15888 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15889 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15890 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15891
15892 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15893 ins_encode %{
15894 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15895 __ string_compare($str1$$Register, $str2$$Register,
15896 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15897 $tmp1$$Register, $tmp2$$Register,
15898 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15899 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15900 StrIntrinsicNode::LU);
15901 %}
15902 ins_pipe(pipe_class_memory);
15903 %}
15904
15905 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15906 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15907 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15908 pRegGov_P1 pgtmp2, rFlagsReg cr)
15909 %{
15910 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15911 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15912 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15913 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15914
15915 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15916 ins_encode %{
15917 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15918 __ string_compare($str1$$Register, $str2$$Register,
15919 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15920 $tmp1$$Register, $tmp2$$Register,
15921 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15922 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15923 StrIntrinsicNode::UL);
15924 %}
15925 ins_pipe(pipe_class_memory);
15926 %}
15927
15928 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15929 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15930 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15931 pRegGov_P1 pgtmp2, rFlagsReg cr)
15932 %{
15933 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15934 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15935 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15936 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15937
15938 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15939 ins_encode %{
15940 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15941 __ string_compare($str1$$Register, $str2$$Register,
15942 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15943 $tmp1$$Register, $tmp2$$Register,
15944 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15945 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15946 StrIntrinsicNode::UU);
15947 %}
15948 ins_pipe(pipe_class_memory);
15949 %}
15950
15951 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15952 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15953 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15954 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15955 %{
15956 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15957 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15958 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15959 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15960 TEMP vtmp0, TEMP vtmp1, KILL cr);
15961 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15962 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15963
15964 ins_encode %{
15965 __ string_indexof($str1$$Register, $str2$$Register,
15966 $cnt1$$Register, $cnt2$$Register,
15967 $tmp1$$Register, $tmp2$$Register,
15968 $tmp3$$Register, $tmp4$$Register,
15969 $tmp5$$Register, $tmp6$$Register,
15970 -1, $result$$Register, StrIntrinsicNode::UU);
15971 %}
15972 ins_pipe(pipe_class_memory);
15973 %}
15974
15975 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15976 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15977 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15978 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15979 %{
15980 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15981 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15982 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15983 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15984 TEMP vtmp0, TEMP vtmp1, KILL cr);
15985 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15986 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15987
15988 ins_encode %{
15989 __ string_indexof($str1$$Register, $str2$$Register,
15990 $cnt1$$Register, $cnt2$$Register,
15991 $tmp1$$Register, $tmp2$$Register,
15992 $tmp3$$Register, $tmp4$$Register,
15993 $tmp5$$Register, $tmp6$$Register,
15994 -1, $result$$Register, StrIntrinsicNode::LL);
15995 %}
15996 ins_pipe(pipe_class_memory);
15997 %}
15998
15999 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16000 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16001 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16002 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16003 %{
16004 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16005 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16006 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16007 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16008 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16009 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16010 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16011
16012 ins_encode %{
16013 __ string_indexof($str1$$Register, $str2$$Register,
16014 $cnt1$$Register, $cnt2$$Register,
16015 $tmp1$$Register, $tmp2$$Register,
16016 $tmp3$$Register, $tmp4$$Register,
16017 $tmp5$$Register, $tmp6$$Register,
16018 -1, $result$$Register, StrIntrinsicNode::UL);
16019 %}
16020 ins_pipe(pipe_class_memory);
16021 %}
16022
16023 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16024 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16025 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16026 %{
16027 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16028 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16029 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16030 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16031 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16032 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16033
16034 ins_encode %{
16035 int icnt2 = (int)$int_cnt2$$constant;
16036 __ string_indexof($str1$$Register, $str2$$Register,
16037 $cnt1$$Register, zr,
16038 $tmp1$$Register, $tmp2$$Register,
16039 $tmp3$$Register, $tmp4$$Register, zr, zr,
16040 icnt2, $result$$Register, StrIntrinsicNode::UU);
16041 %}
16042 ins_pipe(pipe_class_memory);
16043 %}
16044
16045 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16046 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16047 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16048 %{
16049 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16050 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16051 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16052 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16053 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16054 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16055
16056 ins_encode %{
16057 int icnt2 = (int)$int_cnt2$$constant;
16058 __ string_indexof($str1$$Register, $str2$$Register,
16059 $cnt1$$Register, zr,
16060 $tmp1$$Register, $tmp2$$Register,
16061 $tmp3$$Register, $tmp4$$Register, zr, zr,
16062 icnt2, $result$$Register, StrIntrinsicNode::LL);
16063 %}
16064 ins_pipe(pipe_class_memory);
16065 %}
16066
16067 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16068 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16069 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16070 %{
16071 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16072 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16073 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16074 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16075 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16076 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16077
16078 ins_encode %{
16079 int icnt2 = (int)$int_cnt2$$constant;
16080 __ string_indexof($str1$$Register, $str2$$Register,
16081 $cnt1$$Register, zr,
16082 $tmp1$$Register, $tmp2$$Register,
16083 $tmp3$$Register, $tmp4$$Register, zr, zr,
16084 icnt2, $result$$Register, StrIntrinsicNode::UL);
16085 %}
16086 ins_pipe(pipe_class_memory);
16087 %}
16088
16089 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16090 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16091 iRegINoSp tmp3, rFlagsReg cr)
16092 %{
16093 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16094 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16095 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16096 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16097
16098 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16099
16100 ins_encode %{
16101 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16102 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16103 $tmp3$$Register);
16104 %}
16105 ins_pipe(pipe_class_memory);
16106 %}
16107
16108 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16109 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16110 iRegINoSp tmp3, rFlagsReg cr)
16111 %{
16112 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16113 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16114 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16115 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16116
16117 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16118
16119 ins_encode %{
16120 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16121 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16122 $tmp3$$Register);
16123 %}
16124 ins_pipe(pipe_class_memory);
16125 %}
16126
16127 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16128 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16129 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16130 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16131 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16132 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16133 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16134 ins_encode %{
16135 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16136 $result$$Register, $ztmp1$$FloatRegister,
16137 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16138 $ptmp$$PRegister, true /* isL */);
16139 %}
16140 ins_pipe(pipe_class_memory);
16141 %}
16142
16143 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16144 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16145 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16146 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16147 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16148 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16149 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16150 ins_encode %{
16151 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16152 $result$$Register, $ztmp1$$FloatRegister,
16153 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16154 $ptmp$$PRegister, false /* isL */);
16155 %}
16156 ins_pipe(pipe_class_memory);
16157 %}
16158
16159 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16160 iRegI_R0 result, rFlagsReg cr)
16161 %{
16162 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16163 match(Set result (StrEquals (Binary str1 str2) cnt));
16164 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16165
16166 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16167 ins_encode %{
16168 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16169 __ string_equals($str1$$Register, $str2$$Register,
16170 $result$$Register, $cnt$$Register);
16171 %}
16172 ins_pipe(pipe_class_memory);
16173 %}
16174
16175 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16176 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16177 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16178 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16179 iRegP_R10 tmp, rFlagsReg cr)
16180 %{
16181 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16182 match(Set result (AryEq ary1 ary2));
16183 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16184 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16185 TEMP vtmp6, TEMP vtmp7, KILL cr);
16186
16187 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16188 ins_encode %{
16189 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16190 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16191 $result$$Register, $tmp$$Register, 1);
16192 if (tpc == nullptr) {
16193 ciEnv::current()->record_failure("CodeCache is full");
16194 return;
16195 }
16196 %}
16197 ins_pipe(pipe_class_memory);
16198 %}
16199
16200 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16201 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16202 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16203 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16204 iRegP_R10 tmp, rFlagsReg cr)
16205 %{
16206 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16207 match(Set result (AryEq ary1 ary2));
16208 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16209 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16210 TEMP vtmp6, TEMP vtmp7, KILL cr);
16211
16212 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16213 ins_encode %{
16214 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16215 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16216 $result$$Register, $tmp$$Register, 2);
16217 if (tpc == nullptr) {
16218 ciEnv::current()->record_failure("CodeCache is full");
16219 return;
16220 }
16221 %}
16222 ins_pipe(pipe_class_memory);
16223 %}
16224
16225 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16226 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16227 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16228 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16229 %{
16230 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16231 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16232 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16233
16234 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16235 ins_encode %{
16236 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16237 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16238 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16239 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16240 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16241 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16242 (BasicType)$basic_type$$constant);
16243 if (tpc == nullptr) {
16244 ciEnv::current()->record_failure("CodeCache is full");
16245 return;
16246 }
16247 %}
16248 ins_pipe(pipe_class_memory);
16249 %}
16250
16251 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16252 %{
16253 match(Set result (CountPositives ary1 len));
16254 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16255 format %{ "count positives byte[] $ary1,$len -> $result" %}
16256 ins_encode %{
16257 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16258 if (tpc == nullptr) {
16259 ciEnv::current()->record_failure("CodeCache is full");
16260 return;
16261 }
16262 %}
16263 ins_pipe( pipe_slow );
16264 %}
16265
16266 // fast char[] to byte[] compression
16267 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16268 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16269 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16270 iRegI_R0 result, rFlagsReg cr)
16271 %{
16272 match(Set result (StrCompressedCopy src (Binary dst len)));
16273 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16274 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16275
16276 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16277 ins_encode %{
16278 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16279 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16280 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16281 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16282 %}
16283 ins_pipe(pipe_slow);
16284 %}
16285
16286 // fast byte[] to char[] inflation
16287 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16288 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16289 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16290 %{
16291 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16292 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16293 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16294 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16295
16296 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16297 ins_encode %{
16298 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16299 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16300 $vtmp2$$FloatRegister, $tmp$$Register);
16301 if (tpc == nullptr) {
16302 ciEnv::current()->record_failure("CodeCache is full");
16303 return;
16304 }
16305 %}
16306 ins_pipe(pipe_class_memory);
16307 %}
16308
16309 // encode char[] to byte[] in ISO_8859_1
16310 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16311 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16312 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16313 iRegI_R0 result, rFlagsReg cr)
16314 %{
16315 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16316 match(Set result (EncodeISOArray src (Binary dst len)));
16317 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16318 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16319
16320 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16321 ins_encode %{
16322 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16323 $result$$Register, false,
16324 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16325 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16326 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16327 %}
16328 ins_pipe(pipe_class_memory);
16329 %}
16330
16331 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16332 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16333 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16334 iRegI_R0 result, rFlagsReg cr)
16335 %{
16336 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16337 match(Set result (EncodeISOArray src (Binary dst len)));
16338 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16339 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16340
16341 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16342 ins_encode %{
16343 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16344 $result$$Register, true,
16345 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16346 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16347 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16348 %}
16349 ins_pipe(pipe_class_memory);
16350 %}
16351
16352 //----------------------------- CompressBits/ExpandBits ------------------------
16353
16354 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16355 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16356 match(Set dst (CompressBits src mask));
16357 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16358 format %{ "mov $tsrc, $src\n\t"
16359 "mov $tmask, $mask\n\t"
16360 "bext $tdst, $tsrc, $tmask\n\t"
16361 "mov $dst, $tdst"
16362 %}
16363 ins_encode %{
16364 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16365 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16366 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16367 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16368 %}
16369 ins_pipe(pipe_slow);
16370 %}
16371
16372 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16373 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16374 match(Set dst (CompressBits (LoadI mem) mask));
16375 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16376 format %{ "ldrs $tsrc, $mem\n\t"
16377 "ldrs $tmask, $mask\n\t"
16378 "bext $tdst, $tsrc, $tmask\n\t"
16379 "mov $dst, $tdst"
16380 %}
16381 ins_encode %{
16382 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16383 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16384 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16385 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16386 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16387 %}
16388 ins_pipe(pipe_slow);
16389 %}
16390
16391 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16392 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16393 match(Set dst (CompressBits src mask));
16394 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16395 format %{ "mov $tsrc, $src\n\t"
16396 "mov $tmask, $mask\n\t"
16397 "bext $tdst, $tsrc, $tmask\n\t"
16398 "mov $dst, $tdst"
16399 %}
16400 ins_encode %{
16401 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16402 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16403 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16404 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16405 %}
16406 ins_pipe(pipe_slow);
16407 %}
16408
16409 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16410 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16411 match(Set dst (CompressBits (LoadL mem) mask));
16412 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16413 format %{ "ldrd $tsrc, $mem\n\t"
16414 "ldrd $tmask, $mask\n\t"
16415 "bext $tdst, $tsrc, $tmask\n\t"
16416 "mov $dst, $tdst"
16417 %}
16418 ins_encode %{
16419 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16420 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16421 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16422 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16423 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16424 %}
16425 ins_pipe(pipe_slow);
16426 %}
16427
16428 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16429 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16430 match(Set dst (ExpandBits src mask));
16431 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16432 format %{ "mov $tsrc, $src\n\t"
16433 "mov $tmask, $mask\n\t"
16434 "bdep $tdst, $tsrc, $tmask\n\t"
16435 "mov $dst, $tdst"
16436 %}
16437 ins_encode %{
16438 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16439 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16440 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16441 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16442 %}
16443 ins_pipe(pipe_slow);
16444 %}
16445
16446 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16447 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16448 match(Set dst (ExpandBits (LoadI mem) mask));
16449 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16450 format %{ "ldrs $tsrc, $mem\n\t"
16451 "ldrs $tmask, $mask\n\t"
16452 "bdep $tdst, $tsrc, $tmask\n\t"
16453 "mov $dst, $tdst"
16454 %}
16455 ins_encode %{
16456 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16457 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16458 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16459 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16460 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16461 %}
16462 ins_pipe(pipe_slow);
16463 %}
16464
16465 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16466 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16467 match(Set dst (ExpandBits src mask));
16468 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16469 format %{ "mov $tsrc, $src\n\t"
16470 "mov $tmask, $mask\n\t"
16471 "bdep $tdst, $tsrc, $tmask\n\t"
16472 "mov $dst, $tdst"
16473 %}
16474 ins_encode %{
16475 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16476 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16477 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16478 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16479 %}
16480 ins_pipe(pipe_slow);
16481 %}
16482
16483
16484 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16485 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16486 match(Set dst (ExpandBits (LoadL mem) mask));
16487 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16488 format %{ "ldrd $tsrc, $mem\n\t"
16489 "ldrd $tmask, $mask\n\t"
16490 "bdep $tdst, $tsrc, $tmask\n\t"
16491 "mov $dst, $tdst"
16492 %}
16493 ins_encode %{
16494 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16495 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16496 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16497 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16498 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16499 %}
16500 ins_pipe(pipe_slow);
16501 %}
16502
16503 //----------------------------- Reinterpret ----------------------------------
16504 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16505 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16506 match(Set dst (ReinterpretHF2S src));
16507 format %{ "reinterpretHF2S $dst, $src" %}
16508 ins_encode %{
16509 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16510 %}
16511 ins_pipe(pipe_slow);
16512 %}
16513
16514 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16515 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16516 match(Set dst (ReinterpretS2HF src));
16517 format %{ "reinterpretS2HF $dst, $src" %}
16518 ins_encode %{
16519 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16520 %}
16521 ins_pipe(pipe_slow);
16522 %}
16523
16524 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16525 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16526 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16527 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16528 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16529 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16530 // can be omitted in this pattern, resulting in -
16531 // fcvt $dst, $src // Convert float to half-precision float
16532 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16533 %{
16534 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16535 format %{ "convF2HFAndS2HF $dst, $src" %}
16536 ins_encode %{
16537 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16538 %}
16539 ins_pipe(pipe_slow);
16540 %}
16541
16542 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16543 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16544 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16545 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16546 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16547 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16548 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16549 // resulting in -
16550 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16551 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16552 %{
16553 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16554 format %{ "convHF2SAndHF2F $dst, $src" %}
16555 ins_encode %{
16556 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16557 %}
16558 ins_pipe(pipe_slow);
16559 %}
16560
16561 // ============================================================================
16562 // This name is KNOWN by the ADLC and cannot be changed.
16563 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16564 // for this guy.
16565 instruct tlsLoadP(thread_RegP dst)
16566 %{
16567 match(Set dst (ThreadLocal));
16568
16569 ins_cost(0);
16570
16571 format %{ " -- \t// $dst=Thread::current(), empty" %}
16572
16573 size(0);
16574
16575 ins_encode( /*empty*/ );
16576
16577 ins_pipe(pipe_class_empty);
16578 %}
16579
16580 //----------PEEPHOLE RULES-----------------------------------------------------
16581 // These must follow all instruction definitions as they use the names
16582 // defined in the instructions definitions.
16583 //
16584 // peepmatch ( root_instr_name [preceding_instruction]* );
16585 //
16586 // peepconstraint %{
16587 // (instruction_number.operand_name relational_op instruction_number.operand_name
16588 // [, ...] );
16589 // // instruction numbers are zero-based using left to right order in peepmatch
16590 //
16591 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16592 // // provide an instruction_number.operand_name for each operand that appears
16593 // // in the replacement instruction's match rule
16594 //
16595 // ---------VM FLAGS---------------------------------------------------------
16596 //
16597 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16598 //
16599 // Each peephole rule is given an identifying number starting with zero and
16600 // increasing by one in the order seen by the parser. An individual peephole
16601 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16602 // on the command-line.
16603 //
16604 // ---------CURRENT LIMITATIONS----------------------------------------------
16605 //
16606 // Only match adjacent instructions in same basic block
16607 // Only equality constraints
16608 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16609 // Only one replacement instruction
16610 //
16611 // ---------EXAMPLE----------------------------------------------------------
16612 //
16613 // // pertinent parts of existing instructions in architecture description
16614 // instruct movI(iRegINoSp dst, iRegI src)
16615 // %{
16616 // match(Set dst (CopyI src));
16617 // %}
16618 //
16619 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16620 // %{
16621 // match(Set dst (AddI dst src));
16622 // effect(KILL cr);
16623 // %}
16624 //
16625 // // Change (inc mov) to lea
16626 // peephole %{
16627 // // increment preceded by register-register move
16628 // peepmatch ( incI_iReg movI );
16629 // // require that the destination register of the increment
16630 // // match the destination register of the move
16631 // peepconstraint ( 0.dst == 1.dst );
16632 // // construct a replacement instruction that sets
16633 // // the destination to ( move's source register + one )
16634 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16635 // %}
16636 //
16637
16638 // Implementation no longer uses movX instructions since
16639 // machine-independent system no longer uses CopyX nodes.
16640 //
16641 // peephole
16642 // %{
16643 // peepmatch (incI_iReg movI);
16644 // peepconstraint (0.dst == 1.dst);
16645 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16646 // %}
16647
16648 // peephole
16649 // %{
16650 // peepmatch (decI_iReg movI);
16651 // peepconstraint (0.dst == 1.dst);
16652 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16653 // %}
16654
16655 // peephole
16656 // %{
16657 // peepmatch (addI_iReg_imm movI);
16658 // peepconstraint (0.dst == 1.dst);
16659 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16660 // %}
16661
16662 // peephole
16663 // %{
16664 // peepmatch (incL_iReg movL);
16665 // peepconstraint (0.dst == 1.dst);
16666 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16667 // %}
16668
16669 // peephole
16670 // %{
16671 // peepmatch (decL_iReg movL);
16672 // peepconstraint (0.dst == 1.dst);
16673 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16674 // %}
16675
16676 // peephole
16677 // %{
16678 // peepmatch (addL_iReg_imm movL);
16679 // peepconstraint (0.dst == 1.dst);
16680 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16681 // %}
16682
16683 // peephole
16684 // %{
16685 // peepmatch (addP_iReg_imm movP);
16686 // peepconstraint (0.dst == 1.dst);
16687 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16688 // %}
16689
16690 // // Change load of spilled value to only a spill
16691 // instruct storeI(memory mem, iRegI src)
16692 // %{
16693 // match(Set mem (StoreI mem src));
16694 // %}
16695 //
16696 // instruct loadI(iRegINoSp dst, memory mem)
16697 // %{
16698 // match(Set dst (LoadI mem));
16699 // %}
16700 //
16701
16702 //----------SMARTSPILL RULES---------------------------------------------------
16703 // These must follow all instruction definitions as they use the names
16704 // defined in the instructions definitions.
16705
16706 // Local Variables:
16707 // mode: c++
16708 // End: