1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else if (_entry_point == nullptr) {
1694 // See CallLeafNoFPIndirect
1695 return 1 * NativeInstruction::instruction_size;
1696 } else {
1697 return 6 * NativeInstruction::instruction_size;
1698 }
1699 }
1700
1701 //=============================================================================
1702
1703 #ifndef PRODUCT
1704 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1705 st->print("BREAKPOINT");
1706 }
1707 #endif
1708
1709 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1710 __ brk(0);
1711 }
1712
1713 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1714 return MachNode::size(ra_);
1715 }
1716
1717 //=============================================================================
1718
1719 #ifndef PRODUCT
1720 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1721 st->print("nop \t# %d bytes pad for loops and calls", _count);
1722 }
1723 #endif
1724
1725 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1726 for (int i = 0; i < _count; i++) {
1727 __ nop();
1728 }
1729 }
1730
1731 uint MachNopNode::size(PhaseRegAlloc*) const {
1732 return _count * NativeInstruction::instruction_size;
1733 }
1734
1735 //=============================================================================
1736 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1737
1738 int ConstantTable::calculate_table_base_offset() const {
1739 return 0; // absolute addressing, no offset
1740 }
1741
1742 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1743 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1744 ShouldNotReachHere();
1745 }
1746
1747 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1748 // Empty encoding
1749 }
1750
1751 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1752 return 0;
1753 }
1754
1755 #ifndef PRODUCT
1756 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1757 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1758 }
1759 #endif
1760
1761 #ifndef PRODUCT
1762 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1763 Compile* C = ra_->C;
1764
1765 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1766
1767 if (C->output()->need_stack_bang(framesize))
1768 st->print("# stack bang size=%d\n\t", framesize);
1769
1770 if (VM_Version::use_rop_protection()) {
1771 st->print("ldr zr, [lr]\n\t");
1772 st->print("paciaz\n\t");
1773 }
1774 if (framesize < ((1 << 9) + 2 * wordSize)) {
1775 st->print("sub sp, sp, #%d\n\t", framesize);
1776 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1777 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1778 } else {
1779 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1780 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1781 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1782 st->print("sub sp, sp, rscratch1");
1783 }
1784 if (C->stub_function() == nullptr) {
1785 st->print("\n\t");
1786 st->print("ldr rscratch1, [guard]\n\t");
1787 st->print("dmb ishld\n\t");
1788 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1789 st->print("cmp rscratch1, rscratch2\n\t");
1790 st->print("b.eq skip");
1791 st->print("\n\t");
1792 st->print("blr #nmethod_entry_barrier_stub\n\t");
1793 st->print("b skip\n\t");
1794 st->print("guard: int\n\t");
1795 st->print("\n\t");
1796 st->print("skip:\n\t");
1797 }
1798 }
1799 #endif
1800
1801 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1802 Compile* C = ra_->C;
1803
1804
1805 __ verified_entry(C, 0);
1806
1807 if (C->stub_function() == nullptr) {
1808 __ entry_barrier();
1809 }
1810
1811 if (!Compile::current()->output()->in_scratch_emit_size()) {
1812 __ bind(*_verified_entry);
1813 }
1814
1815 if (VerifyStackAtCalls) {
1816 Unimplemented();
1817 }
1818
1819 C->output()->set_frame_complete(__ offset());
1820
1821 if (C->has_mach_constant_base_node()) {
1822 // NOTE: We set the table base offset here because users might be
1823 // emitted before MachConstantBaseNode.
1824 ConstantTable& constant_table = C->output()->constant_table();
1825 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1826 }
1827 }
1828
1829 int MachPrologNode::reloc() const
1830 {
1831 return 0;
1832 }
1833
1834 //=============================================================================
1835
1836 #ifndef PRODUCT
1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1838 Compile* C = ra_->C;
1839 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1840
1841 st->print("# pop frame %d\n\t",framesize);
1842
1843 if (framesize == 0) {
1844 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1845 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1846 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1847 st->print("add sp, sp, #%d\n\t", framesize);
1848 } else {
1849 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1850 st->print("add sp, sp, rscratch1\n\t");
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 }
1853 if (VM_Version::use_rop_protection()) {
1854 st->print("autiaz\n\t");
1855 st->print("ldr zr, [lr]\n\t");
1856 }
1857
1858 if (do_polling() && C->is_method_compilation()) {
1859 st->print("# test polling word\n\t");
1860 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1861 st->print("cmp sp, rscratch1\n\t");
1862 st->print("bhi #slow_path");
1863 }
1864 }
1865 #endif
1866
1867 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1868 Compile* C = ra_->C;
1869 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1870
1871 __ remove_frame(framesize, C->needs_stack_repair());
1872
1873 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1874 __ reserved_stack_check();
1875 }
1876
1877 if (do_polling() && C->is_method_compilation()) {
1878 Label dummy_label;
1879 Label* code_stub = &dummy_label;
1880 if (!C->output()->in_scratch_emit_size()) {
1881 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1882 C->output()->add_stub(stub);
1883 code_stub = &stub->entry();
1884 }
1885 __ relocate(relocInfo::poll_return_type);
1886 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1887 }
1888 }
1889
1890 int MachEpilogNode::reloc() const {
1891 // Return number of relocatable values contained in this instruction.
1892 return 1; // 1 for polling page.
1893 }
1894
1895 const Pipeline * MachEpilogNode::pipeline() const {
1896 return MachNode::pipeline_class();
1897 }
1898
1899 //=============================================================================
1900
1901 static enum RC rc_class(OptoReg::Name reg) {
1902
1903 if (reg == OptoReg::Bad) {
1904 return rc_bad;
1905 }
1906
1907 // we have 32 int registers * 2 halves
1908 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1909
1910 if (reg < slots_of_int_registers) {
1911 return rc_int;
1912 }
1913
1914 // we have 32 float register * 8 halves
1915 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1916 if (reg < slots_of_int_registers + slots_of_float_registers) {
1917 return rc_float;
1918 }
1919
1920 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1921 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1922 return rc_predicate;
1923 }
1924
1925 // Between predicate regs & stack is the flags.
1926 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1927
1928 return rc_stack;
1929 }
1930
1931 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1932 Compile* C = ra_->C;
1933
1934 // Get registers to move.
1935 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1936 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1937 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1938 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1939
1940 enum RC src_hi_rc = rc_class(src_hi);
1941 enum RC src_lo_rc = rc_class(src_lo);
1942 enum RC dst_hi_rc = rc_class(dst_hi);
1943 enum RC dst_lo_rc = rc_class(dst_lo);
1944
1945 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1946
1947 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1948 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1949 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1950 "expected aligned-adjacent pairs");
1951 }
1952
1953 if (src_lo == dst_lo && src_hi == dst_hi) {
1954 return 0; // Self copy, no move.
1955 }
1956
1957 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1958 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1959 int src_offset = ra_->reg2offset(src_lo);
1960 int dst_offset = ra_->reg2offset(dst_lo);
1961
1962 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1963 uint ireg = ideal_reg();
1964 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
1965 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
1966 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
1967 if (ireg == Op_VecA && masm) {
1968 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1969 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1970 // stack->stack
1971 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1972 sve_vector_reg_size_in_bytes);
1973 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1974 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1975 sve_vector_reg_size_in_bytes);
1976 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1977 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1978 sve_vector_reg_size_in_bytes);
1979 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1980 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1981 as_FloatRegister(Matcher::_regEncode[src_lo]),
1982 as_FloatRegister(Matcher::_regEncode[src_lo]));
1983 } else {
1984 ShouldNotReachHere();
1985 }
1986 } else if (masm) {
1987 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1988 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1989 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1990 // stack->stack
1991 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1992 if (ireg == Op_VecD) {
1993 __ unspill(rscratch1, true, src_offset);
1994 __ spill(rscratch1, true, dst_offset);
1995 } else {
1996 __ spill_copy128(src_offset, dst_offset);
1997 }
1998 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1999 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2000 ireg == Op_VecD ? __ T8B : __ T16B,
2001 as_FloatRegister(Matcher::_regEncode[src_lo]));
2002 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2003 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2004 ireg == Op_VecD ? __ D : __ Q,
2005 ra_->reg2offset(dst_lo));
2006 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2007 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2008 ireg == Op_VecD ? __ D : __ Q,
2009 ra_->reg2offset(src_lo));
2010 } else {
2011 ShouldNotReachHere();
2012 }
2013 }
2014 } else if (masm) {
2015 switch (src_lo_rc) {
2016 case rc_int:
2017 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2018 if (is64) {
2019 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2020 as_Register(Matcher::_regEncode[src_lo]));
2021 } else {
2022 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2023 as_Register(Matcher::_regEncode[src_lo]));
2024 }
2025 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2026 if (is64) {
2027 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2028 as_Register(Matcher::_regEncode[src_lo]));
2029 } else {
2030 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2031 as_Register(Matcher::_regEncode[src_lo]));
2032 }
2033 } else { // gpr --> stack spill
2034 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2035 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2036 }
2037 break;
2038 case rc_float:
2039 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2040 if (is64) {
2041 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2042 as_FloatRegister(Matcher::_regEncode[src_lo]));
2043 } else {
2044 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2045 as_FloatRegister(Matcher::_regEncode[src_lo]));
2046 }
2047 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2048 if (is64) {
2049 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 as_FloatRegister(Matcher::_regEncode[src_lo]));
2051 } else {
2052 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2053 as_FloatRegister(Matcher::_regEncode[src_lo]));
2054 }
2055 } else { // fpr --> stack spill
2056 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2057 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2058 is64 ? __ D : __ S, dst_offset);
2059 }
2060 break;
2061 case rc_stack:
2062 if (dst_lo_rc == rc_int) { // stack --> gpr load
2063 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2064 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2065 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2066 is64 ? __ D : __ S, src_offset);
2067 } else if (dst_lo_rc == rc_predicate) {
2068 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2069 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2070 } else { // stack --> stack copy
2071 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2072 if (ideal_reg() == Op_RegVectMask) {
2073 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2074 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2075 } else {
2076 __ unspill(rscratch1, is64, src_offset);
2077 __ spill(rscratch1, is64, dst_offset);
2078 }
2079 }
2080 break;
2081 case rc_predicate:
2082 if (dst_lo_rc == rc_predicate) {
2083 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2084 } else if (dst_lo_rc == rc_stack) {
2085 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2086 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2087 } else {
2088 assert(false, "bad src and dst rc_class combination.");
2089 ShouldNotReachHere();
2090 }
2091 break;
2092 default:
2093 assert(false, "bad rc_class for spill");
2094 ShouldNotReachHere();
2095 }
2096 }
2097
2098 if (st) {
2099 st->print("spill ");
2100 if (src_lo_rc == rc_stack) {
2101 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2102 } else {
2103 st->print("%s -> ", Matcher::regName[src_lo]);
2104 }
2105 if (dst_lo_rc == rc_stack) {
2106 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2107 } else {
2108 st->print("%s", Matcher::regName[dst_lo]);
2109 }
2110 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2111 int vsize = 0;
2112 switch (ideal_reg()) {
2113 case Op_VecD:
2114 vsize = 64;
2115 break;
2116 case Op_VecX:
2117 vsize = 128;
2118 break;
2119 case Op_VecA:
2120 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2121 break;
2122 default:
2123 assert(false, "bad register type for spill");
2124 ShouldNotReachHere();
2125 }
2126 st->print("\t# vector spill size = %d", vsize);
2127 } else if (ideal_reg() == Op_RegVectMask) {
2128 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2129 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2130 st->print("\t# predicate spill size = %d", vsize);
2131 } else {
2132 st->print("\t# spill size = %d", is64 ? 64 : 32);
2133 }
2134 }
2135
2136 return 0;
2137
2138 }
2139
2140 #ifndef PRODUCT
2141 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2142 if (!ra_)
2143 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2144 else
2145 implementation(nullptr, ra_, false, st);
2146 }
2147 #endif
2148
2149 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2150 implementation(masm, ra_, false, nullptr);
2151 }
2152
2153 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2154 return MachNode::size(ra_);
2155 }
2156
2157 //=============================================================================
2158
2159 #ifndef PRODUCT
2160 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2161 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2162 int reg = ra_->get_reg_first(this);
2163 st->print("add %s, rsp, #%d]\t# box lock",
2164 Matcher::regName[reg], offset);
2165 }
2166 #endif
2167
2168 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2169 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2170 int reg = ra_->get_encode(this);
2171
2172 // This add will handle any 24-bit signed offset. 24 bits allows an
2173 // 8 megabyte stack frame.
2174 __ add(as_Register(reg), sp, offset);
2175 }
2176
2177 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2178 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2179 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2180
2181 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2182 return NativeInstruction::instruction_size;
2183 } else {
2184 return 2 * NativeInstruction::instruction_size;
2185 }
2186 }
2187
2188 ///=============================================================================
2189 #ifndef PRODUCT
2190 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2191 {
2192 st->print_cr("# MachVEPNode");
2193 if (!_verified) {
2194 st->print_cr("\t load_class");
2195 } else {
2196 st->print_cr("\t unpack_inline_arg");
2197 }
2198 }
2199 #endif
2200
2201 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2202 {
2203 if (!_verified) {
2204 __ ic_check(1);
2205 } else {
2206 // TODO 8284443 Avoid creation of temporary frame
2207 if (ra_->C->stub_function() == nullptr) {
2208 __ verified_entry(ra_->C, 0);
2209 __ entry_barrier();
2210 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2211 __ remove_frame(framesize, false);
2212 }
2213 // Unpack inline type args passed as oop and then jump to
2214 // the verified entry point (skipping the unverified entry).
2215 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2216 // Emit code for verified entry and save increment for stack repair on return
2217 __ verified_entry(ra_->C, sp_inc);
2218 if (Compile::current()->output()->in_scratch_emit_size()) {
2219 Label dummy_verified_entry;
2220 __ b(dummy_verified_entry);
2221 } else {
2222 __ b(*_verified_entry);
2223 }
2224 }
2225 }
2226
2227 //=============================================================================
2228 #ifndef PRODUCT
2229 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2230 {
2231 st->print_cr("# MachUEPNode");
2232 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2233 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2234 st->print_cr("\tcmpw rscratch1, r10");
2235 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2236 }
2237 #endif
2238
2239 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2240 {
2241 __ ic_check(InteriorEntryAlignment);
2242 }
2243
2244 // REQUIRED EMIT CODE
2245
2246 //=============================================================================
2247
2248 // Emit deopt handler code.
2249 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2250 {
2251 // Note that the code buffer's insts_mark is always relative to insts.
2252 // That's why we must use the macroassembler to generate a handler.
2253 address base = __ start_a_stub(size_deopt_handler());
2254 if (base == nullptr) {
2255 ciEnv::current()->record_failure("CodeCache is full");
2256 return 0; // CodeBuffer::expand failed
2257 }
2258
2259 int offset = __ offset();
2260 Label start;
2261 __ bind(start);
2262 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2263
2264 int entry_offset = __ offset();
2265 __ b(start);
2266
2267 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2268 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2269 "out of bounds read in post-call NOP check");
2270 __ end_a_stub();
2271 return entry_offset;
2272 }
2273
2274 // REQUIRED MATCHER CODE
2275
2276 //=============================================================================
2277
2278 bool Matcher::match_rule_supported(int opcode) {
2279 if (!has_match_rule(opcode))
2280 return false;
2281
2282 switch (opcode) {
2283 case Op_OnSpinWait:
2284 return VM_Version::supports_on_spin_wait();
2285 case Op_CacheWB:
2286 case Op_CacheWBPreSync:
2287 case Op_CacheWBPostSync:
2288 if (!VM_Version::supports_data_cache_line_flush()) {
2289 return false;
2290 }
2291 break;
2292 case Op_ExpandBits:
2293 case Op_CompressBits:
2294 if (!VM_Version::supports_svebitperm()) {
2295 return false;
2296 }
2297 break;
2298 case Op_FmaF:
2299 case Op_FmaD:
2300 case Op_FmaVF:
2301 case Op_FmaVD:
2302 if (!UseFMA) {
2303 return false;
2304 }
2305 break;
2306 case Op_FmaHF:
2307 // UseFMA flag also needs to be checked along with FEAT_FP16
2308 if (!UseFMA || !is_feat_fp16_supported()) {
2309 return false;
2310 }
2311 break;
2312 case Op_AddHF:
2313 case Op_SubHF:
2314 case Op_MulHF:
2315 case Op_DivHF:
2316 case Op_MinHF:
2317 case Op_MaxHF:
2318 case Op_SqrtHF:
2319 // Half-precision floating point scalar operations require FEAT_FP16
2320 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2321 // features are supported.
2322 if (!is_feat_fp16_supported()) {
2323 return false;
2324 }
2325 break;
2326 }
2327
2328 return true; // Per default match rules are supported.
2329 }
2330
2331 const RegMask* Matcher::predicate_reg_mask(void) {
2332 return &_PR_REG_mask;
2333 }
2334
2335 bool Matcher::supports_vector_calling_convention(void) {
2336 return EnableVectorSupport;
2337 }
2338
2339 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2340 assert(EnableVectorSupport, "sanity");
2341 int lo = V0_num;
2342 int hi = V0_H_num;
2343 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2344 hi = V0_K_num;
2345 }
2346 return OptoRegPair(hi, lo);
2347 }
2348
2349 // Is this branch offset short enough that a short branch can be used?
2350 //
2351 // NOTE: If the platform does not provide any short branch variants, then
2352 // this method should return false for offset 0.
2353 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2354 // The passed offset is relative to address of the branch.
2355
2356 return (-32768 <= offset && offset < 32768);
2357 }
2358
2359 // Vector width in bytes.
2360 int Matcher::vector_width_in_bytes(BasicType bt) {
2361 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2362 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2363 // Minimum 2 values in vector
2364 if (size < 2*type2aelembytes(bt)) size = 0;
2365 // But never < 4
2366 if (size < 4) size = 0;
2367 return size;
2368 }
2369
2370 // Limits on vector size (number of elements) loaded into vector.
2371 int Matcher::max_vector_size(const BasicType bt) {
2372 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2373 }
2374
2375 int Matcher::min_vector_size(const BasicType bt) {
2376 // Usually, the shortest vector length supported by AArch64 ISA and
2377 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2378 // vectors in a few special cases.
2379 int size;
2380 switch(bt) {
2381 case T_BOOLEAN:
2382 // Load/store a vector mask with only 2 elements for vector types
2383 // such as "2I/2F/2L/2D".
2384 size = 2;
2385 break;
2386 case T_BYTE:
2387 // Generate a "4B" vector, to support vector cast between "8B/16B"
2388 // and "4S/4I/4L/4F/4D".
2389 size = 4;
2390 break;
2391 case T_SHORT:
2392 // Generate a "2S" vector, to support vector cast between "4S/8S"
2393 // and "2I/2L/2F/2D".
2394 size = 2;
2395 break;
2396 default:
2397 // Limit the min vector length to 64-bit.
2398 size = 8 / type2aelembytes(bt);
2399 // The number of elements in a vector should be at least 2.
2400 size = MAX2(size, 2);
2401 }
2402
2403 int max_size = max_vector_size(bt);
2404 return MIN2(size, max_size);
2405 }
2406
2407 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2408 return Matcher::max_vector_size(bt);
2409 }
2410
2411 // Actual max scalable vector register length.
2412 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2413 return Matcher::max_vector_size(bt);
2414 }
2415
2416 // Vector ideal reg.
2417 uint Matcher::vector_ideal_reg(int len) {
2418 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2419 return Op_VecA;
2420 }
2421 switch(len) {
2422 // For 16-bit/32-bit mask vector, reuse VecD.
2423 case 2:
2424 case 4:
2425 case 8: return Op_VecD;
2426 case 16: return Op_VecX;
2427 }
2428 ShouldNotReachHere();
2429 return 0;
2430 }
2431
2432 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2433 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2434 switch (ideal_reg) {
2435 case Op_VecA: return new vecAOper();
2436 case Op_VecD: return new vecDOper();
2437 case Op_VecX: return new vecXOper();
2438 }
2439 ShouldNotReachHere();
2440 return nullptr;
2441 }
2442
2443 bool Matcher::is_reg2reg_move(MachNode* m) {
2444 return false;
2445 }
2446
2447 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2448 return false;
2449 }
2450
2451 bool Matcher::is_generic_vector(MachOper* opnd) {
2452 return opnd->opcode() == VREG;
2453 }
2454
2455 #ifdef ASSERT
2456 // Return whether or not this register is ever used as an argument.
2457 bool Matcher::can_be_java_arg(int reg)
2458 {
2459 return
2460 reg == R0_num || reg == R0_H_num ||
2461 reg == R1_num || reg == R1_H_num ||
2462 reg == R2_num || reg == R2_H_num ||
2463 reg == R3_num || reg == R3_H_num ||
2464 reg == R4_num || reg == R4_H_num ||
2465 reg == R5_num || reg == R5_H_num ||
2466 reg == R6_num || reg == R6_H_num ||
2467 reg == R7_num || reg == R7_H_num ||
2468 reg == V0_num || reg == V0_H_num ||
2469 reg == V1_num || reg == V1_H_num ||
2470 reg == V2_num || reg == V2_H_num ||
2471 reg == V3_num || reg == V3_H_num ||
2472 reg == V4_num || reg == V4_H_num ||
2473 reg == V5_num || reg == V5_H_num ||
2474 reg == V6_num || reg == V6_H_num ||
2475 reg == V7_num || reg == V7_H_num;
2476 }
2477 #endif
2478
2479 uint Matcher::int_pressure_limit()
2480 {
2481 // JDK-8183543: When taking the number of available registers as int
2482 // register pressure threshold, the jtreg test:
2483 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2484 // failed due to C2 compilation failure with
2485 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2486 //
2487 // A derived pointer is live at CallNode and then is flagged by RA
2488 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2489 // derived pointers and lastly fail to spill after reaching maximum
2490 // number of iterations. Lowering the default pressure threshold to
2491 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2492 // a high register pressure area of the code so that split_DEF can
2493 // generate DefinitionSpillCopy for the derived pointer.
2494 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2495 if (!PreserveFramePointer) {
2496 // When PreserveFramePointer is off, frame pointer is allocatable,
2497 // but different from other SOC registers, it is excluded from
2498 // fatproj's mask because its save type is No-Save. Decrease 1 to
2499 // ensure high pressure at fatproj when PreserveFramePointer is off.
2500 // See check_pressure_at_fatproj().
2501 default_int_pressure_threshold--;
2502 }
2503 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2504 }
2505
2506 uint Matcher::float_pressure_limit()
2507 {
2508 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2509 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2510 }
2511
2512 const RegMask& Matcher::divI_proj_mask() {
2513 ShouldNotReachHere();
2514 return RegMask::EMPTY;
2515 }
2516
2517 // Register for MODI projection of divmodI.
2518 const RegMask& Matcher::modI_proj_mask() {
2519 ShouldNotReachHere();
2520 return RegMask::EMPTY;
2521 }
2522
2523 // Register for DIVL projection of divmodL.
2524 const RegMask& Matcher::divL_proj_mask() {
2525 ShouldNotReachHere();
2526 return RegMask::EMPTY;
2527 }
2528
2529 // Register for MODL projection of divmodL.
2530 const RegMask& Matcher::modL_proj_mask() {
2531 ShouldNotReachHere();
2532 return RegMask::EMPTY;
2533 }
2534
2535 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2536 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2537 Node* u = addp->fast_out(i);
2538 if (u->is_LoadStore()) {
2539 // On AArch64, LoadStoreNodes (i.e. compare and swap
2540 // instructions) only take register indirect as an operand, so
2541 // any attempt to use an AddPNode as an input to a LoadStoreNode
2542 // must fail.
2543 return false;
2544 }
2545 if (u->is_Mem()) {
2546 int opsize = u->as_Mem()->memory_size();
2547 assert(opsize > 0, "unexpected memory operand size");
2548 if (u->as_Mem()->memory_size() != (1<<shift)) {
2549 return false;
2550 }
2551 }
2552 }
2553 return true;
2554 }
2555
2556 // Convert BoolTest condition to Assembler condition.
2557 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2558 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2559 Assembler::Condition result;
2560 switch(cond) {
2561 case BoolTest::eq:
2562 result = Assembler::EQ; break;
2563 case BoolTest::ne:
2564 result = Assembler::NE; break;
2565 case BoolTest::le:
2566 result = Assembler::LE; break;
2567 case BoolTest::ge:
2568 result = Assembler::GE; break;
2569 case BoolTest::lt:
2570 result = Assembler::LT; break;
2571 case BoolTest::gt:
2572 result = Assembler::GT; break;
2573 case BoolTest::ule:
2574 result = Assembler::LS; break;
2575 case BoolTest::uge:
2576 result = Assembler::HS; break;
2577 case BoolTest::ult:
2578 result = Assembler::LO; break;
2579 case BoolTest::ugt:
2580 result = Assembler::HI; break;
2581 case BoolTest::overflow:
2582 result = Assembler::VS; break;
2583 case BoolTest::no_overflow:
2584 result = Assembler::VC; break;
2585 default:
2586 ShouldNotReachHere();
2587 return Assembler::Condition(-1);
2588 }
2589
2590 // Check conversion
2591 if (cond & BoolTest::unsigned_compare) {
2592 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2593 } else {
2594 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2595 }
2596
2597 return result;
2598 }
2599
2600 // Binary src (Replicate con)
2601 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2602 if (n == nullptr || m == nullptr) {
2603 return false;
2604 }
2605
2606 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2607 return false;
2608 }
2609
2610 Node* imm_node = m->in(1);
2611 if (!imm_node->is_Con()) {
2612 return false;
2613 }
2614
2615 const Type* t = imm_node->bottom_type();
2616 if (!(t->isa_int() || t->isa_long())) {
2617 return false;
2618 }
2619
2620 switch (n->Opcode()) {
2621 case Op_AndV:
2622 case Op_OrV:
2623 case Op_XorV: {
2624 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2625 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2626 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2627 }
2628 case Op_AddVB:
2629 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2630 case Op_AddVS:
2631 case Op_AddVI:
2632 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2633 case Op_AddVL:
2634 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2635 default:
2636 return false;
2637 }
2638 }
2639
2640 // (XorV src (Replicate m1))
2641 // (XorVMask src (MaskAll m1))
2642 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2643 if (n != nullptr && m != nullptr) {
2644 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2645 VectorNode::is_all_ones_vector(m);
2646 }
2647 return false;
2648 }
2649
2650 // Should the matcher clone input 'm' of node 'n'?
2651 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2652 if (is_vshift_con_pattern(n, m) ||
2653 is_vector_bitwise_not_pattern(n, m) ||
2654 is_valid_sve_arith_imm_pattern(n, m) ||
2655 is_encode_and_store_pattern(n, m)) {
2656 mstack.push(m, Visit);
2657 return true;
2658 }
2659 return false;
2660 }
2661
2662 // Should the Matcher clone shifts on addressing modes, expecting them
2663 // to be subsumed into complex addressing expressions or compute them
2664 // into registers?
2665 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2666
2667 // Loads and stores with indirect memory input (e.g., volatile loads and
2668 // stores) do not subsume the input into complex addressing expressions. If
2669 // the addressing expression is input to at least one such load or store, do
2670 // not clone the addressing expression. Query needs_acquiring_load and
2671 // needs_releasing_store as a proxy for indirect memory input, as it is not
2672 // possible to directly query for indirect memory input at this stage.
2673 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2674 Node* n = m->fast_out(i);
2675 if (n->is_Load() && needs_acquiring_load(n)) {
2676 return false;
2677 }
2678 if (n->is_Store() && needs_releasing_store(n)) {
2679 return false;
2680 }
2681 }
2682
2683 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2684 return true;
2685 }
2686
2687 Node *off = m->in(AddPNode::Offset);
2688 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2689 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2690 // Are there other uses besides address expressions?
2691 !is_visited(off)) {
2692 address_visited.set(off->_idx); // Flag as address_visited
2693 mstack.push(off->in(2), Visit);
2694 Node *conv = off->in(1);
2695 if (conv->Opcode() == Op_ConvI2L &&
2696 // Are there other uses besides address expressions?
2697 !is_visited(conv)) {
2698 address_visited.set(conv->_idx); // Flag as address_visited
2699 mstack.push(conv->in(1), Pre_Visit);
2700 } else {
2701 mstack.push(conv, Pre_Visit);
2702 }
2703 address_visited.test_set(m->_idx); // Flag as address_visited
2704 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2705 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2706 return true;
2707 } else if (off->Opcode() == Op_ConvI2L &&
2708 // Are there other uses besides address expressions?
2709 !is_visited(off)) {
2710 address_visited.test_set(m->_idx); // Flag as address_visited
2711 address_visited.set(off->_idx); // Flag as address_visited
2712 mstack.push(off->in(1), Pre_Visit);
2713 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2714 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2715 return true;
2716 }
2717 return false;
2718 }
2719
2720 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2721 { \
2722 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2723 guarantee(DISP == 0, "mode not permitted for volatile"); \
2724 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2725 __ INSN(REG, as_Register(BASE)); \
2726 }
2727
2728
2729 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2730 {
2731 Address::extend scale;
2732
2733 // Hooboy, this is fugly. We need a way to communicate to the
2734 // encoder that the index needs to be sign extended, so we have to
2735 // enumerate all the cases.
2736 switch (opcode) {
2737 case INDINDEXSCALEDI2L:
2738 case INDINDEXSCALEDI2LN:
2739 case INDINDEXI2L:
2740 case INDINDEXI2LN:
2741 scale = Address::sxtw(size);
2742 break;
2743 default:
2744 scale = Address::lsl(size);
2745 }
2746
2747 if (index == -1) {
2748 return Address(base, disp);
2749 } else {
2750 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2751 return Address(base, as_Register(index), scale);
2752 }
2753 }
2754
2755
2756 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2757 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2758 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2759 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2760 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2761
2762 // Used for all non-volatile memory accesses. The use of
2763 // $mem->opcode() to discover whether this pattern uses sign-extended
2764 // offsets is something of a kludge.
2765 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2766 Register reg, int opcode,
2767 Register base, int index, int scale, int disp,
2768 int size_in_memory)
2769 {
2770 Address addr = mem2address(opcode, base, index, scale, disp);
2771 if (addr.getMode() == Address::base_plus_offset) {
2772 /* Fix up any out-of-range offsets. */
2773 assert_different_registers(rscratch1, base);
2774 assert_different_registers(rscratch1, reg);
2775 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2776 }
2777 (masm->*insn)(reg, addr);
2778 }
2779
2780 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2781 FloatRegister reg, int opcode,
2782 Register base, int index, int size, int disp,
2783 int size_in_memory)
2784 {
2785 Address::extend scale;
2786
2787 switch (opcode) {
2788 case INDINDEXSCALEDI2L:
2789 case INDINDEXSCALEDI2LN:
2790 scale = Address::sxtw(size);
2791 break;
2792 default:
2793 scale = Address::lsl(size);
2794 }
2795
2796 if (index == -1) {
2797 // Fix up any out-of-range offsets.
2798 assert_different_registers(rscratch1, base);
2799 Address addr = Address(base, disp);
2800 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2801 (masm->*insn)(reg, addr);
2802 } else {
2803 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2804 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2805 }
2806 }
2807
2808 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2809 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2810 int opcode, Register base, int index, int size, int disp)
2811 {
2812 if (index == -1) {
2813 (masm->*insn)(reg, T, Address(base, disp));
2814 } else {
2815 assert(disp == 0, "unsupported address mode");
2816 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2817 }
2818 }
2819
2820 %}
2821
2822
2823
2824 //----------ENCODING BLOCK-----------------------------------------------------
2825 // This block specifies the encoding classes used by the compiler to
2826 // output byte streams. Encoding classes are parameterized macros
2827 // used by Machine Instruction Nodes in order to generate the bit
2828 // encoding of the instruction. Operands specify their base encoding
2829 // interface with the interface keyword. There are currently
2830 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2831 // COND_INTER. REG_INTER causes an operand to generate a function
2832 // which returns its register number when queried. CONST_INTER causes
2833 // an operand to generate a function which returns the value of the
2834 // constant when queried. MEMORY_INTER causes an operand to generate
2835 // four functions which return the Base Register, the Index Register,
2836 // the Scale Value, and the Offset Value of the operand when queried.
2837 // COND_INTER causes an operand to generate six functions which return
2838 // the encoding code (ie - encoding bits for the instruction)
2839 // associated with each basic boolean condition for a conditional
2840 // instruction.
2841 //
2842 // Instructions specify two basic values for encoding. Again, a
2843 // function is available to check if the constant displacement is an
2844 // oop. They use the ins_encode keyword to specify their encoding
2845 // classes (which must be a sequence of enc_class names, and their
2846 // parameters, specified in the encoding block), and they use the
2847 // opcode keyword to specify, in order, their primary, secondary, and
2848 // tertiary opcode. Only the opcode sections which a particular
2849 // instruction needs for encoding need to be specified.
2850 encode %{
2851 // Build emit functions for each basic byte or larger field in the
2852 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2853 // from C++ code in the enc_class source block. Emit functions will
2854 // live in the main source block for now. In future, we can
2855 // generalize this by adding a syntax that specifies the sizes of
2856 // fields in an order, so that the adlc can build the emit functions
2857 // automagically
2858
2859 // catch all for unimplemented encodings
2860 enc_class enc_unimplemented %{
2861 __ unimplemented("C2 catch all");
2862 %}
2863
2864 // BEGIN Non-volatile memory access
2865
2866 // This encoding class is generated automatically from ad_encode.m4.
2867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2868 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2869 Register dst_reg = as_Register($dst$$reg);
2870 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2871 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2872 %}
2873
2874 // This encoding class is generated automatically from ad_encode.m4.
2875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2876 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2877 Register dst_reg = as_Register($dst$$reg);
2878 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2879 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2880 %}
2881
2882 // This encoding class is generated automatically from ad_encode.m4.
2883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2884 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2885 Register dst_reg = as_Register($dst$$reg);
2886 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2887 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2888 %}
2889
2890 // This encoding class is generated automatically from ad_encode.m4.
2891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2892 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2893 Register dst_reg = as_Register($dst$$reg);
2894 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2895 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2896 %}
2897
2898 // This encoding class is generated automatically from ad_encode.m4.
2899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2900 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2901 Register dst_reg = as_Register($dst$$reg);
2902 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2903 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2904 %}
2905
2906 // This encoding class is generated automatically from ad_encode.m4.
2907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2908 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2909 Register dst_reg = as_Register($dst$$reg);
2910 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2911 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2912 %}
2913
2914 // This encoding class is generated automatically from ad_encode.m4.
2915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2916 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2917 Register dst_reg = as_Register($dst$$reg);
2918 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2919 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2920 %}
2921
2922 // This encoding class is generated automatically from ad_encode.m4.
2923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2924 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2925 Register dst_reg = as_Register($dst$$reg);
2926 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2927 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2928 %}
2929
2930 // This encoding class is generated automatically from ad_encode.m4.
2931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2932 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2933 Register dst_reg = as_Register($dst$$reg);
2934 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2935 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2936 %}
2937
2938 // This encoding class is generated automatically from ad_encode.m4.
2939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2940 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2941 Register dst_reg = as_Register($dst$$reg);
2942 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2943 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2944 %}
2945
2946 // This encoding class is generated automatically from ad_encode.m4.
2947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2948 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2949 Register dst_reg = as_Register($dst$$reg);
2950 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2951 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2952 %}
2953
2954 // This encoding class is generated automatically from ad_encode.m4.
2955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2956 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2957 Register dst_reg = as_Register($dst$$reg);
2958 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2960 %}
2961
2962 // This encoding class is generated automatically from ad_encode.m4.
2963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2964 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2965 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2966 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2967 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2968 %}
2969
2970 // This encoding class is generated automatically from ad_encode.m4.
2971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2972 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2973 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2974 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2975 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2976 %}
2977
2978 // This encoding class is generated automatically from ad_encode.m4.
2979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2980 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2981 Register src_reg = as_Register($src$$reg);
2982 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2983 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2984 %}
2985
2986 // This encoding class is generated automatically from ad_encode.m4.
2987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2988 enc_class aarch64_enc_strb0(memory1 mem) %{
2989 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2990 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2991 %}
2992
2993 // This encoding class is generated automatically from ad_encode.m4.
2994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2995 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2996 Register src_reg = as_Register($src$$reg);
2997 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2998 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2999 %}
3000
3001 // This encoding class is generated automatically from ad_encode.m4.
3002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3003 enc_class aarch64_enc_strh0(memory2 mem) %{
3004 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3005 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3006 %}
3007
3008 // This encoding class is generated automatically from ad_encode.m4.
3009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3010 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3011 Register src_reg = as_Register($src$$reg);
3012 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3013 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3014 %}
3015
3016 // This encoding class is generated automatically from ad_encode.m4.
3017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3018 enc_class aarch64_enc_strw0(memory4 mem) %{
3019 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3020 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3021 %}
3022
3023 // This encoding class is generated automatically from ad_encode.m4.
3024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3025 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3026 Register src_reg = as_Register($src$$reg);
3027 // we sometimes get asked to store the stack pointer into the
3028 // current thread -- we cannot do that directly on AArch64
3029 if (src_reg == r31_sp) {
3030 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3031 __ mov(rscratch2, sp);
3032 src_reg = rscratch2;
3033 }
3034 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3035 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3036 %}
3037
3038 // This encoding class is generated automatically from ad_encode.m4.
3039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3040 enc_class aarch64_enc_str0(memory8 mem) %{
3041 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3042 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3043 %}
3044
3045 // This encoding class is generated automatically from ad_encode.m4.
3046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3047 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3048 FloatRegister src_reg = as_FloatRegister($src$$reg);
3049 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3050 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3051 %}
3052
3053 // This encoding class is generated automatically from ad_encode.m4.
3054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3055 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3056 FloatRegister src_reg = as_FloatRegister($src$$reg);
3057 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3058 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3059 %}
3060
3061 // This encoding class is generated automatically from ad_encode.m4.
3062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3063 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3064 __ membar(Assembler::StoreStore);
3065 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3066 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3067 %}
3068
3069 // END Non-volatile memory access
3070
3071 // Vector loads and stores
3072 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3073 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3074 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3075 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3076 %}
3077
3078 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3079 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3080 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3081 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3082 %}
3083
3084 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3085 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3086 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3087 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3088 %}
3089
3090 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3091 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3092 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3093 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3094 %}
3095
3096 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3097 FloatRegister src_reg = as_FloatRegister($src$$reg);
3098 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3099 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3100 %}
3101
3102 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3103 FloatRegister src_reg = as_FloatRegister($src$$reg);
3104 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3105 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3106 %}
3107
3108 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3109 FloatRegister src_reg = as_FloatRegister($src$$reg);
3110 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3111 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3112 %}
3113
3114 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3115 FloatRegister src_reg = as_FloatRegister($src$$reg);
3116 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3117 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3118 %}
3119
3120 // volatile loads and stores
3121
3122 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3123 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3124 rscratch1, stlrb);
3125 %}
3126
3127 enc_class aarch64_enc_stlrb0(memory mem) %{
3128 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3129 rscratch1, stlrb);
3130 %}
3131
3132 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3133 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3134 rscratch1, stlrh);
3135 %}
3136
3137 enc_class aarch64_enc_stlrh0(memory mem) %{
3138 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3139 rscratch1, stlrh);
3140 %}
3141
3142 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3143 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3144 rscratch1, stlrw);
3145 %}
3146
3147 enc_class aarch64_enc_stlrw0(memory mem) %{
3148 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, stlrw);
3150 %}
3151
3152 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3153 Register dst_reg = as_Register($dst$$reg);
3154 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, ldarb);
3156 __ sxtbw(dst_reg, dst_reg);
3157 %}
3158
3159 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3160 Register dst_reg = as_Register($dst$$reg);
3161 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3162 rscratch1, ldarb);
3163 __ sxtb(dst_reg, dst_reg);
3164 %}
3165
3166 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3167 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3168 rscratch1, ldarb);
3169 %}
3170
3171 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3172 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3173 rscratch1, ldarb);
3174 %}
3175
3176 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3177 Register dst_reg = as_Register($dst$$reg);
3178 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, ldarh);
3180 __ sxthw(dst_reg, dst_reg);
3181 %}
3182
3183 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3184 Register dst_reg = as_Register($dst$$reg);
3185 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3186 rscratch1, ldarh);
3187 __ sxth(dst_reg, dst_reg);
3188 %}
3189
3190 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3191 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3192 rscratch1, ldarh);
3193 %}
3194
3195 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3196 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3197 rscratch1, ldarh);
3198 %}
3199
3200 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3201 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3202 rscratch1, ldarw);
3203 %}
3204
3205 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3206 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3207 rscratch1, ldarw);
3208 %}
3209
3210 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3211 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, ldar);
3213 %}
3214
3215 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3216 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3217 rscratch1, ldarw);
3218 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3219 %}
3220
3221 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3222 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3223 rscratch1, ldar);
3224 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3225 %}
3226
3227 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3228 Register src_reg = as_Register($src$$reg);
3229 // we sometimes get asked to store the stack pointer into the
3230 // current thread -- we cannot do that directly on AArch64
3231 if (src_reg == r31_sp) {
3232 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3233 __ mov(rscratch2, sp);
3234 src_reg = rscratch2;
3235 }
3236 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3237 rscratch1, stlr);
3238 %}
3239
3240 enc_class aarch64_enc_stlr0(memory mem) %{
3241 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3242 rscratch1, stlr);
3243 %}
3244
3245 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3246 {
3247 FloatRegister src_reg = as_FloatRegister($src$$reg);
3248 __ fmovs(rscratch2, src_reg);
3249 }
3250 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3251 rscratch1, stlrw);
3252 %}
3253
3254 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3255 {
3256 FloatRegister src_reg = as_FloatRegister($src$$reg);
3257 __ fmovd(rscratch2, src_reg);
3258 }
3259 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3260 rscratch1, stlr);
3261 %}
3262
3263 // synchronized read/update encodings
3264
3265 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3266 Register dst_reg = as_Register($dst$$reg);
3267 Register base = as_Register($mem$$base);
3268 int index = $mem$$index;
3269 int scale = $mem$$scale;
3270 int disp = $mem$$disp;
3271 if (index == -1) {
3272 if (disp != 0) {
3273 __ lea(rscratch1, Address(base, disp));
3274 __ ldaxr(dst_reg, rscratch1);
3275 } else {
3276 // TODO
3277 // should we ever get anything other than this case?
3278 __ ldaxr(dst_reg, base);
3279 }
3280 } else {
3281 Register index_reg = as_Register(index);
3282 if (disp == 0) {
3283 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3284 __ ldaxr(dst_reg, rscratch1);
3285 } else {
3286 __ lea(rscratch1, Address(base, disp));
3287 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3288 __ ldaxr(dst_reg, rscratch1);
3289 }
3290 }
3291 %}
3292
3293 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3294 Register src_reg = as_Register($src$$reg);
3295 Register base = as_Register($mem$$base);
3296 int index = $mem$$index;
3297 int scale = $mem$$scale;
3298 int disp = $mem$$disp;
3299 if (index == -1) {
3300 if (disp != 0) {
3301 __ lea(rscratch2, Address(base, disp));
3302 __ stlxr(rscratch1, src_reg, rscratch2);
3303 } else {
3304 // TODO
3305 // should we ever get anything other than this case?
3306 __ stlxr(rscratch1, src_reg, base);
3307 }
3308 } else {
3309 Register index_reg = as_Register(index);
3310 if (disp == 0) {
3311 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3312 __ stlxr(rscratch1, src_reg, rscratch2);
3313 } else {
3314 __ lea(rscratch2, Address(base, disp));
3315 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3316 __ stlxr(rscratch1, src_reg, rscratch2);
3317 }
3318 }
3319 __ cmpw(rscratch1, zr);
3320 %}
3321
3322 // prefetch encodings
3323
3324 enc_class aarch64_enc_prefetchw(memory mem) %{
3325 Register base = as_Register($mem$$base);
3326 int index = $mem$$index;
3327 int scale = $mem$$scale;
3328 int disp = $mem$$disp;
3329 if (index == -1) {
3330 // Fix up any out-of-range offsets.
3331 assert_different_registers(rscratch1, base);
3332 Address addr = Address(base, disp);
3333 addr = __ legitimize_address(addr, 8, rscratch1);
3334 __ prfm(addr, PSTL1KEEP);
3335 } else {
3336 Register index_reg = as_Register(index);
3337 if (disp == 0) {
3338 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3339 } else {
3340 __ lea(rscratch1, Address(base, disp));
3341 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3342 }
3343 }
3344 %}
3345
3346 // mov encodings
3347
3348 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3349 uint32_t con = (uint32_t)$src$$constant;
3350 Register dst_reg = as_Register($dst$$reg);
3351 if (con == 0) {
3352 __ movw(dst_reg, zr);
3353 } else {
3354 __ movw(dst_reg, con);
3355 }
3356 %}
3357
3358 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3359 Register dst_reg = as_Register($dst$$reg);
3360 uint64_t con = (uint64_t)$src$$constant;
3361 if (con == 0) {
3362 __ mov(dst_reg, zr);
3363 } else {
3364 __ mov(dst_reg, con);
3365 }
3366 %}
3367
3368 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3369 Register dst_reg = as_Register($dst$$reg);
3370 address con = (address)$src$$constant;
3371 if (con == nullptr || con == (address)1) {
3372 ShouldNotReachHere();
3373 } else {
3374 relocInfo::relocType rtype = $src->constant_reloc();
3375 if (rtype == relocInfo::oop_type) {
3376 __ movoop(dst_reg, (jobject)con);
3377 } else if (rtype == relocInfo::metadata_type) {
3378 __ mov_metadata(dst_reg, (Metadata*)con);
3379 } else {
3380 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3381 // load fake address constants using a normal move
3382 if (! __ is_valid_AArch64_address(con) ||
3383 con < (address)(uintptr_t)os::vm_page_size()) {
3384 __ mov(dst_reg, con);
3385 } else {
3386 // no reloc so just use adrp and add
3387 uint64_t offset;
3388 __ adrp(dst_reg, con, offset);
3389 __ add(dst_reg, dst_reg, offset);
3390 }
3391 }
3392 }
3393 %}
3394
3395 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3396 Register dst_reg = as_Register($dst$$reg);
3397 __ mov(dst_reg, zr);
3398 %}
3399
3400 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3401 Register dst_reg = as_Register($dst$$reg);
3402 __ mov(dst_reg, (uint64_t)1);
3403 %}
3404
3405 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3406 Register dst_reg = as_Register($dst$$reg);
3407 address con = (address)$src$$constant;
3408 if (con == nullptr) {
3409 ShouldNotReachHere();
3410 } else {
3411 relocInfo::relocType rtype = $src->constant_reloc();
3412 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3413 __ set_narrow_oop(dst_reg, (jobject)con);
3414 }
3415 %}
3416
3417 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3418 Register dst_reg = as_Register($dst$$reg);
3419 __ mov(dst_reg, zr);
3420 %}
3421
3422 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3423 Register dst_reg = as_Register($dst$$reg);
3424 address con = (address)$src$$constant;
3425 if (con == nullptr) {
3426 ShouldNotReachHere();
3427 } else {
3428 relocInfo::relocType rtype = $src->constant_reloc();
3429 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3430 __ set_narrow_klass(dst_reg, (Klass *)con);
3431 }
3432 %}
3433
3434 // arithmetic encodings
3435
3436 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3437 Register dst_reg = as_Register($dst$$reg);
3438 Register src_reg = as_Register($src1$$reg);
3439 int32_t con = (int32_t)$src2$$constant;
3440 // add has primary == 0, subtract has primary == 1
3441 if ($primary) { con = -con; }
3442 if (con < 0) {
3443 __ subw(dst_reg, src_reg, -con);
3444 } else {
3445 __ addw(dst_reg, src_reg, con);
3446 }
3447 %}
3448
3449 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3450 Register dst_reg = as_Register($dst$$reg);
3451 Register src_reg = as_Register($src1$$reg);
3452 int32_t con = (int32_t)$src2$$constant;
3453 // add has primary == 0, subtract has primary == 1
3454 if ($primary) { con = -con; }
3455 if (con < 0) {
3456 __ sub(dst_reg, src_reg, -con);
3457 } else {
3458 __ add(dst_reg, src_reg, con);
3459 }
3460 %}
3461
3462 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3463 Register dst_reg = as_Register($dst$$reg);
3464 Register src1_reg = as_Register($src1$$reg);
3465 Register src2_reg = as_Register($src2$$reg);
3466 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3467 %}
3468
3469 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3470 Register dst_reg = as_Register($dst$$reg);
3471 Register src1_reg = as_Register($src1$$reg);
3472 Register src2_reg = as_Register($src2$$reg);
3473 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3474 %}
3475
3476 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3477 Register dst_reg = as_Register($dst$$reg);
3478 Register src1_reg = as_Register($src1$$reg);
3479 Register src2_reg = as_Register($src2$$reg);
3480 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3481 %}
3482
3483 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3484 Register dst_reg = as_Register($dst$$reg);
3485 Register src1_reg = as_Register($src1$$reg);
3486 Register src2_reg = as_Register($src2$$reg);
3487 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3488 %}
3489
3490 // compare instruction encodings
3491
3492 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3493 Register reg1 = as_Register($src1$$reg);
3494 Register reg2 = as_Register($src2$$reg);
3495 __ cmpw(reg1, reg2);
3496 %}
3497
3498 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3499 Register reg = as_Register($src1$$reg);
3500 int32_t val = $src2$$constant;
3501 if (val >= 0) {
3502 __ subsw(zr, reg, val);
3503 } else {
3504 __ addsw(zr, reg, -val);
3505 }
3506 %}
3507
3508 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3509 Register reg1 = as_Register($src1$$reg);
3510 uint32_t val = (uint32_t)$src2$$constant;
3511 __ movw(rscratch1, val);
3512 __ cmpw(reg1, rscratch1);
3513 %}
3514
3515 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3516 Register reg1 = as_Register($src1$$reg);
3517 Register reg2 = as_Register($src2$$reg);
3518 __ cmp(reg1, reg2);
3519 %}
3520
3521 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3522 Register reg = as_Register($src1$$reg);
3523 int64_t val = $src2$$constant;
3524 if (val >= 0) {
3525 __ subs(zr, reg, val);
3526 } else if (val != -val) {
3527 __ adds(zr, reg, -val);
3528 } else {
3529 // aargh, Long.MIN_VALUE is a special case
3530 __ orr(rscratch1, zr, (uint64_t)val);
3531 __ subs(zr, reg, rscratch1);
3532 }
3533 %}
3534
3535 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3536 Register reg1 = as_Register($src1$$reg);
3537 uint64_t val = (uint64_t)$src2$$constant;
3538 __ mov(rscratch1, val);
3539 __ cmp(reg1, rscratch1);
3540 %}
3541
3542 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3543 Register reg1 = as_Register($src1$$reg);
3544 Register reg2 = as_Register($src2$$reg);
3545 __ cmp(reg1, reg2);
3546 %}
3547
3548 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3549 Register reg1 = as_Register($src1$$reg);
3550 Register reg2 = as_Register($src2$$reg);
3551 __ cmpw(reg1, reg2);
3552 %}
3553
3554 enc_class aarch64_enc_testp(iRegP src) %{
3555 Register reg = as_Register($src$$reg);
3556 __ cmp(reg, zr);
3557 %}
3558
3559 enc_class aarch64_enc_testn(iRegN src) %{
3560 Register reg = as_Register($src$$reg);
3561 __ cmpw(reg, zr);
3562 %}
3563
3564 enc_class aarch64_enc_b(label lbl) %{
3565 Label *L = $lbl$$label;
3566 __ b(*L);
3567 %}
3568
3569 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3570 Label *L = $lbl$$label;
3571 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3572 %}
3573
3574 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3575 Label *L = $lbl$$label;
3576 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3577 %}
3578
3579 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3580 %{
3581 Register sub_reg = as_Register($sub$$reg);
3582 Register super_reg = as_Register($super$$reg);
3583 Register temp_reg = as_Register($temp$$reg);
3584 Register result_reg = as_Register($result$$reg);
3585
3586 Label miss;
3587 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3588 nullptr, &miss,
3589 /*set_cond_codes:*/ true);
3590 if ($primary) {
3591 __ mov(result_reg, zr);
3592 }
3593 __ bind(miss);
3594 %}
3595
3596 enc_class aarch64_enc_java_static_call(method meth) %{
3597 address addr = (address)$meth$$method;
3598 address call;
3599 if (!_method) {
3600 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3601 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3602 if (call == nullptr) {
3603 ciEnv::current()->record_failure("CodeCache is full");
3604 return;
3605 }
3606 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3607 // The NOP here is purely to ensure that eliding a call to
3608 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3609 __ nop();
3610 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3611 } else {
3612 int method_index = resolved_method_index(masm);
3613 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3614 : static_call_Relocation::spec(method_index);
3615 call = __ trampoline_call(Address(addr, rspec));
3616 if (call == nullptr) {
3617 ciEnv::current()->record_failure("CodeCache is full");
3618 return;
3619 }
3620 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3621 // Calls of the same statically bound method can share
3622 // a stub to the interpreter.
3623 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3624 } else {
3625 // Emit stub for static call
3626 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3627 if (stub == nullptr) {
3628 ciEnv::current()->record_failure("CodeCache is full");
3629 return;
3630 }
3631 }
3632 }
3633
3634 __ post_call_nop();
3635
3636 // Only non uncommon_trap calls need to reinitialize ptrue.
3637 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3638 __ reinitialize_ptrue();
3639 }
3640 %}
3641
3642 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3643 int method_index = resolved_method_index(masm);
3644 address call = __ ic_call((address)$meth$$method, method_index);
3645 if (call == nullptr) {
3646 ciEnv::current()->record_failure("CodeCache is full");
3647 return;
3648 }
3649 __ post_call_nop();
3650 if (Compile::current()->max_vector_size() > 0) {
3651 __ reinitialize_ptrue();
3652 }
3653 %}
3654
3655 enc_class aarch64_enc_call_epilog() %{
3656 if (VerifyStackAtCalls) {
3657 // Check that stack depth is unchanged: find majik cookie on stack
3658 __ call_Unimplemented();
3659 }
3660 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3661 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3662 // Search for the corresponding projection, get the register and emit code that initialized it.
3663 uint con = (tf()->range_cc()->cnt() - 1);
3664 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3665 ProjNode* proj = fast_out(i)->as_Proj();
3666 if (proj->_con == con) {
3667 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3668 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3669 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3670 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3671 __ cmp(r0, zr);
3672 __ cset(toReg, Assembler::NE);
3673 if (reg->is_stack()) {
3674 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3675 __ str(toReg, Address(sp, st_off));
3676 }
3677 break;
3678 }
3679 }
3680 if (return_value_is_used()) {
3681 // An inline type is returned as fields in multiple registers.
3682 // R0 either contains an oop if the inline type is buffered or a pointer
3683 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3684 // if the lowest bit is set to allow C2 to use the oop after null checking.
3685 // r0 &= (r0 & 1) - 1
3686 __ andr(rscratch1, r0, 0x1);
3687 __ sub(rscratch1, rscratch1, 0x1);
3688 __ andr(r0, r0, rscratch1);
3689 }
3690 }
3691 %}
3692
3693 enc_class aarch64_enc_java_to_runtime(method meth) %{
3694 // some calls to generated routines (arraycopy code) are scheduled
3695 // by C2 as runtime calls. if so we can call them using a br (they
3696 // will be in a reachable segment) otherwise we have to use a blr
3697 // which loads the absolute address into a register.
3698 address entry = (address)$meth$$method;
3699 CodeBlob *cb = CodeCache::find_blob(entry);
3700 if (cb) {
3701 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3702 if (call == nullptr) {
3703 ciEnv::current()->record_failure("CodeCache is full");
3704 return;
3705 }
3706 __ post_call_nop();
3707 } else {
3708 Label retaddr;
3709 // Make the anchor frame walkable
3710 __ adr(rscratch2, retaddr);
3711 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3712 __ lea(rscratch1, RuntimeAddress(entry));
3713 __ blr(rscratch1);
3714 __ bind(retaddr);
3715 __ post_call_nop();
3716 }
3717 if (Compile::current()->max_vector_size() > 0) {
3718 __ reinitialize_ptrue();
3719 }
3720 %}
3721
3722 enc_class aarch64_enc_rethrow() %{
3723 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3724 %}
3725
3726 enc_class aarch64_enc_ret() %{
3727 #ifdef ASSERT
3728 if (Compile::current()->max_vector_size() > 0) {
3729 __ verify_ptrue();
3730 }
3731 #endif
3732 __ ret(lr);
3733 %}
3734
3735 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3736 Register target_reg = as_Register($jump_target$$reg);
3737 __ br(target_reg);
3738 %}
3739
3740 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3741 Register target_reg = as_Register($jump_target$$reg);
3742 // exception oop should be in r0
3743 // ret addr has been popped into lr
3744 // callee expects it in r3
3745 __ mov(r3, lr);
3746 __ br(target_reg);
3747 %}
3748
3749 %}
3750
3751 //----------FRAME--------------------------------------------------------------
3752 // Definition of frame structure and management information.
3753 //
3754 // S T A C K L A Y O U T Allocators stack-slot number
3755 // | (to get allocators register number
3756 // G Owned by | | v add OptoReg::stack0())
3757 // r CALLER | |
3758 // o | +--------+ pad to even-align allocators stack-slot
3759 // w V | pad0 | numbers; owned by CALLER
3760 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3761 // h ^ | in | 5
3762 // | | args | 4 Holes in incoming args owned by SELF
3763 // | | | | 3
3764 // | | +--------+
3765 // V | | old out| Empty on Intel, window on Sparc
3766 // | old |preserve| Must be even aligned.
3767 // | SP-+--------+----> Matcher::_old_SP, even aligned
3768 // | | in | 3 area for Intel ret address
3769 // Owned by |preserve| Empty on Sparc.
3770 // SELF +--------+
3771 // | | pad2 | 2 pad to align old SP
3772 // | +--------+ 1
3773 // | | locks | 0
3774 // | +--------+----> OptoReg::stack0(), even aligned
3775 // | | pad1 | 11 pad to align new SP
3776 // | +--------+
3777 // | | | 10
3778 // | | spills | 9 spills
3779 // V | | 8 (pad0 slot for callee)
3780 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3781 // ^ | out | 7
3782 // | | args | 6 Holes in outgoing args owned by CALLEE
3783 // Owned by +--------+
3784 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3785 // | new |preserve| Must be even-aligned.
3786 // | SP-+--------+----> Matcher::_new_SP, even aligned
3787 // | | |
3788 //
3789 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3790 // known from SELF's arguments and the Java calling convention.
3791 // Region 6-7 is determined per call site.
3792 // Note 2: If the calling convention leaves holes in the incoming argument
3793 // area, those holes are owned by SELF. Holes in the outgoing area
3794 // are owned by the CALLEE. Holes should not be necessary in the
3795 // incoming area, as the Java calling convention is completely under
3796 // the control of the AD file. Doubles can be sorted and packed to
3797 // avoid holes. Holes in the outgoing arguments may be necessary for
3798 // varargs C calling conventions.
3799 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3800 // even aligned with pad0 as needed.
3801 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3802 // (the latter is true on Intel but is it false on AArch64?)
3803 // region 6-11 is even aligned; it may be padded out more so that
3804 // the region from SP to FP meets the minimum stack alignment.
3805 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3806 // alignment. Region 11, pad1, may be dynamically extended so that
3807 // SP meets the minimum alignment.
3808
3809 frame %{
3810 // These three registers define part of the calling convention
3811 // between compiled code and the interpreter.
3812
3813 // Inline Cache Register or Method for I2C.
3814 inline_cache_reg(R12);
3815
3816 // Number of stack slots consumed by locking an object
3817 sync_stack_slots(2);
3818
3819 // Compiled code's Frame Pointer
3820 frame_pointer(R31);
3821
3822 // Stack alignment requirement
3823 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3824
3825 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3826 // for calls to C. Supports the var-args backing area for register parms.
3827 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3828
3829 // The after-PROLOG location of the return address. Location of
3830 // return address specifies a type (REG or STACK) and a number
3831 // representing the register number (i.e. - use a register name) or
3832 // stack slot.
3833 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3834 // Otherwise, it is above the locks and verification slot and alignment word
3835 // TODO this may well be correct but need to check why that - 2 is there
3836 // ppc port uses 0 but we definitely need to allow for fixed_slots
3837 // which folds in the space used for monitors
3838 return_addr(STACK - 2 +
3839 align_up((Compile::current()->in_preserve_stack_slots() +
3840 Compile::current()->fixed_slots()),
3841 stack_alignment_in_slots()));
3842
3843 // Location of compiled Java return values. Same as C for now.
3844 return_value
3845 %{
3846 // TODO do we allow ideal_reg == Op_RegN???
3847 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3848 "only return normal values");
3849
3850 static const int lo[Op_RegL + 1] = { // enum name
3851 0, // Op_Node
3852 0, // Op_Set
3853 R0_num, // Op_RegN
3854 R0_num, // Op_RegI
3855 R0_num, // Op_RegP
3856 V0_num, // Op_RegF
3857 V0_num, // Op_RegD
3858 R0_num // Op_RegL
3859 };
3860
3861 static const int hi[Op_RegL + 1] = { // enum name
3862 0, // Op_Node
3863 0, // Op_Set
3864 OptoReg::Bad, // Op_RegN
3865 OptoReg::Bad, // Op_RegI
3866 R0_H_num, // Op_RegP
3867 OptoReg::Bad, // Op_RegF
3868 V0_H_num, // Op_RegD
3869 R0_H_num // Op_RegL
3870 };
3871
3872 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3873 %}
3874 %}
3875
3876 //----------ATTRIBUTES---------------------------------------------------------
3877 //----------Operand Attributes-------------------------------------------------
3878 op_attrib op_cost(1); // Required cost attribute
3879
3880 //----------Instruction Attributes---------------------------------------------
3881 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3882 ins_attrib ins_size(32); // Required size attribute (in bits)
3883 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3884 // a non-matching short branch variant
3885 // of some long branch?
3886 ins_attrib ins_alignment(4); // Required alignment attribute (must
3887 // be a power of 2) specifies the
3888 // alignment that some part of the
3889 // instruction (not necessarily the
3890 // start) requires. If > 1, a
3891 // compute_padding() function must be
3892 // provided for the instruction
3893
3894 // Whether this node is expanded during code emission into a sequence of
3895 // instructions and the first instruction can perform an implicit null check.
3896 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3897
3898 //----------OPERANDS-----------------------------------------------------------
3899 // Operand definitions must precede instruction definitions for correct parsing
3900 // in the ADLC because operands constitute user defined types which are used in
3901 // instruction definitions.
3902
3903 //----------Simple Operands----------------------------------------------------
3904
3905 // Integer operands 32 bit
3906 // 32 bit immediate
3907 operand immI()
3908 %{
3909 match(ConI);
3910
3911 op_cost(0);
3912 format %{ %}
3913 interface(CONST_INTER);
3914 %}
3915
3916 // 32 bit zero
3917 operand immI0()
3918 %{
3919 predicate(n->get_int() == 0);
3920 match(ConI);
3921
3922 op_cost(0);
3923 format %{ %}
3924 interface(CONST_INTER);
3925 %}
3926
3927 // 32 bit unit increment
3928 operand immI_1()
3929 %{
3930 predicate(n->get_int() == 1);
3931 match(ConI);
3932
3933 op_cost(0);
3934 format %{ %}
3935 interface(CONST_INTER);
3936 %}
3937
3938 // 32 bit unit decrement
3939 operand immI_M1()
3940 %{
3941 predicate(n->get_int() == -1);
3942 match(ConI);
3943
3944 op_cost(0);
3945 format %{ %}
3946 interface(CONST_INTER);
3947 %}
3948
3949 // Shift values for add/sub extension shift
3950 operand immIExt()
3951 %{
3952 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3953 match(ConI);
3954
3955 op_cost(0);
3956 format %{ %}
3957 interface(CONST_INTER);
3958 %}
3959
3960 operand immI_gt_1()
3961 %{
3962 predicate(n->get_int() > 1);
3963 match(ConI);
3964
3965 op_cost(0);
3966 format %{ %}
3967 interface(CONST_INTER);
3968 %}
3969
3970 operand immI_le_4()
3971 %{
3972 predicate(n->get_int() <= 4);
3973 match(ConI);
3974
3975 op_cost(0);
3976 format %{ %}
3977 interface(CONST_INTER);
3978 %}
3979
3980 operand immI_4()
3981 %{
3982 predicate(n->get_int() == 4);
3983 match(ConI);
3984
3985 op_cost(0);
3986 format %{ %}
3987 interface(CONST_INTER);
3988 %}
3989
3990 operand immI_16()
3991 %{
3992 predicate(n->get_int() == 16);
3993 match(ConI);
3994
3995 op_cost(0);
3996 format %{ %}
3997 interface(CONST_INTER);
3998 %}
3999
4000 operand immI_24()
4001 %{
4002 predicate(n->get_int() == 24);
4003 match(ConI);
4004
4005 op_cost(0);
4006 format %{ %}
4007 interface(CONST_INTER);
4008 %}
4009
4010 operand immI_32()
4011 %{
4012 predicate(n->get_int() == 32);
4013 match(ConI);
4014
4015 op_cost(0);
4016 format %{ %}
4017 interface(CONST_INTER);
4018 %}
4019
4020 operand immI_48()
4021 %{
4022 predicate(n->get_int() == 48);
4023 match(ConI);
4024
4025 op_cost(0);
4026 format %{ %}
4027 interface(CONST_INTER);
4028 %}
4029
4030 operand immI_56()
4031 %{
4032 predicate(n->get_int() == 56);
4033 match(ConI);
4034
4035 op_cost(0);
4036 format %{ %}
4037 interface(CONST_INTER);
4038 %}
4039
4040 operand immI_255()
4041 %{
4042 predicate(n->get_int() == 255);
4043 match(ConI);
4044
4045 op_cost(0);
4046 format %{ %}
4047 interface(CONST_INTER);
4048 %}
4049
4050 operand immI_65535()
4051 %{
4052 predicate(n->get_int() == 65535);
4053 match(ConI);
4054
4055 op_cost(0);
4056 format %{ %}
4057 interface(CONST_INTER);
4058 %}
4059
4060 operand immI_positive()
4061 %{
4062 predicate(n->get_int() > 0);
4063 match(ConI);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 // BoolTest condition for signed compare
4071 operand immI_cmp_cond()
4072 %{
4073 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4074 match(ConI);
4075
4076 op_cost(0);
4077 format %{ %}
4078 interface(CONST_INTER);
4079 %}
4080
4081 // BoolTest condition for unsigned compare
4082 operand immI_cmpU_cond()
4083 %{
4084 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4085 match(ConI);
4086
4087 op_cost(0);
4088 format %{ %}
4089 interface(CONST_INTER);
4090 %}
4091
4092 operand immL_255()
4093 %{
4094 predicate(n->get_long() == 255L);
4095 match(ConL);
4096
4097 op_cost(0);
4098 format %{ %}
4099 interface(CONST_INTER);
4100 %}
4101
4102 operand immL_65535()
4103 %{
4104 predicate(n->get_long() == 65535L);
4105 match(ConL);
4106
4107 op_cost(0);
4108 format %{ %}
4109 interface(CONST_INTER);
4110 %}
4111
4112 operand immL_4294967295()
4113 %{
4114 predicate(n->get_long() == 4294967295L);
4115 match(ConL);
4116
4117 op_cost(0);
4118 format %{ %}
4119 interface(CONST_INTER);
4120 %}
4121
4122 operand immL_bitmask()
4123 %{
4124 predicate((n->get_long() != 0)
4125 && ((n->get_long() & 0xc000000000000000l) == 0)
4126 && is_power_of_2(n->get_long() + 1));
4127 match(ConL);
4128
4129 op_cost(0);
4130 format %{ %}
4131 interface(CONST_INTER);
4132 %}
4133
4134 operand immI_bitmask()
4135 %{
4136 predicate((n->get_int() != 0)
4137 && ((n->get_int() & 0xc0000000) == 0)
4138 && is_power_of_2(n->get_int() + 1));
4139 match(ConI);
4140
4141 op_cost(0);
4142 format %{ %}
4143 interface(CONST_INTER);
4144 %}
4145
4146 operand immL_positive_bitmaskI()
4147 %{
4148 predicate((n->get_long() != 0)
4149 && ((julong)n->get_long() < 0x80000000ULL)
4150 && is_power_of_2(n->get_long() + 1));
4151 match(ConL);
4152
4153 op_cost(0);
4154 format %{ %}
4155 interface(CONST_INTER);
4156 %}
4157
4158 // Scale values for scaled offset addressing modes (up to long but not quad)
4159 operand immIScale()
4160 %{
4161 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4162 match(ConI);
4163
4164 op_cost(0);
4165 format %{ %}
4166 interface(CONST_INTER);
4167 %}
4168
4169 // 5 bit signed integer
4170 operand immI5()
4171 %{
4172 predicate(Assembler::is_simm(n->get_int(), 5));
4173 match(ConI);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 // 7 bit unsigned integer
4181 operand immIU7()
4182 %{
4183 predicate(Assembler::is_uimm(n->get_int(), 7));
4184 match(ConI);
4185
4186 op_cost(0);
4187 format %{ %}
4188 interface(CONST_INTER);
4189 %}
4190
4191 // Offset for scaled or unscaled immediate loads and stores
4192 operand immIOffset()
4193 %{
4194 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4195 match(ConI);
4196
4197 op_cost(0);
4198 format %{ %}
4199 interface(CONST_INTER);
4200 %}
4201
4202 operand immIOffset1()
4203 %{
4204 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4205 match(ConI);
4206
4207 op_cost(0);
4208 format %{ %}
4209 interface(CONST_INTER);
4210 %}
4211
4212 operand immIOffset2()
4213 %{
4214 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4215 match(ConI);
4216
4217 op_cost(0);
4218 format %{ %}
4219 interface(CONST_INTER);
4220 %}
4221
4222 operand immIOffset4()
4223 %{
4224 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4225 match(ConI);
4226
4227 op_cost(0);
4228 format %{ %}
4229 interface(CONST_INTER);
4230 %}
4231
4232 operand immIOffset8()
4233 %{
4234 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4235 match(ConI);
4236
4237 op_cost(0);
4238 format %{ %}
4239 interface(CONST_INTER);
4240 %}
4241
4242 operand immIOffset16()
4243 %{
4244 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4245 match(ConI);
4246
4247 op_cost(0);
4248 format %{ %}
4249 interface(CONST_INTER);
4250 %}
4251
4252 operand immLOffset()
4253 %{
4254 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4255 match(ConL);
4256
4257 op_cost(0);
4258 format %{ %}
4259 interface(CONST_INTER);
4260 %}
4261
4262 operand immLoffset1()
4263 %{
4264 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4265 match(ConL);
4266
4267 op_cost(0);
4268 format %{ %}
4269 interface(CONST_INTER);
4270 %}
4271
4272 operand immLoffset2()
4273 %{
4274 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4275 match(ConL);
4276
4277 op_cost(0);
4278 format %{ %}
4279 interface(CONST_INTER);
4280 %}
4281
4282 operand immLoffset4()
4283 %{
4284 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4285 match(ConL);
4286
4287 op_cost(0);
4288 format %{ %}
4289 interface(CONST_INTER);
4290 %}
4291
4292 operand immLoffset8()
4293 %{
4294 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4295 match(ConL);
4296
4297 op_cost(0);
4298 format %{ %}
4299 interface(CONST_INTER);
4300 %}
4301
4302 operand immLoffset16()
4303 %{
4304 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4305 match(ConL);
4306
4307 op_cost(0);
4308 format %{ %}
4309 interface(CONST_INTER);
4310 %}
4311
4312 // 5 bit signed long integer
4313 operand immL5()
4314 %{
4315 predicate(Assembler::is_simm(n->get_long(), 5));
4316 match(ConL);
4317
4318 op_cost(0);
4319 format %{ %}
4320 interface(CONST_INTER);
4321 %}
4322
4323 // 7 bit unsigned long integer
4324 operand immLU7()
4325 %{
4326 predicate(Assembler::is_uimm(n->get_long(), 7));
4327 match(ConL);
4328
4329 op_cost(0);
4330 format %{ %}
4331 interface(CONST_INTER);
4332 %}
4333
4334 // 8 bit signed value.
4335 operand immI8()
4336 %{
4337 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4338 match(ConI);
4339
4340 op_cost(0);
4341 format %{ %}
4342 interface(CONST_INTER);
4343 %}
4344
4345 // 8 bit signed value (simm8), or #simm8 LSL 8.
4346 operand immIDupV()
4347 %{
4348 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4349 match(ConI);
4350
4351 op_cost(0);
4352 format %{ %}
4353 interface(CONST_INTER);
4354 %}
4355
4356 // 8 bit signed value (simm8), or #simm8 LSL 8.
4357 operand immLDupV()
4358 %{
4359 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4360 match(ConL);
4361
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 // 8 bit signed value (simm8), or #simm8 LSL 8.
4368 operand immHDupV()
4369 %{
4370 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4371 match(ConH);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 // 8 bit integer valid for vector add sub immediate
4379 operand immBAddSubV()
4380 %{
4381 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4382 match(ConI);
4383
4384 op_cost(0);
4385 format %{ %}
4386 interface(CONST_INTER);
4387 %}
4388
4389 // 32 bit integer valid for add sub immediate
4390 operand immIAddSub()
4391 %{
4392 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4393 match(ConI);
4394 op_cost(0);
4395 format %{ %}
4396 interface(CONST_INTER);
4397 %}
4398
4399 // 32 bit integer valid for vector add sub immediate
4400 operand immIAddSubV()
4401 %{
4402 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4403 match(ConI);
4404
4405 op_cost(0);
4406 format %{ %}
4407 interface(CONST_INTER);
4408 %}
4409
4410 // 32 bit unsigned integer valid for logical immediate
4411
4412 operand immBLog()
4413 %{
4414 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4415 match(ConI);
4416
4417 op_cost(0);
4418 format %{ %}
4419 interface(CONST_INTER);
4420 %}
4421
4422 operand immSLog()
4423 %{
4424 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4425 match(ConI);
4426
4427 op_cost(0);
4428 format %{ %}
4429 interface(CONST_INTER);
4430 %}
4431
4432 operand immILog()
4433 %{
4434 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4435 match(ConI);
4436
4437 op_cost(0);
4438 format %{ %}
4439 interface(CONST_INTER);
4440 %}
4441
4442 // Integer operands 64 bit
4443 // 64 bit immediate
4444 operand immL()
4445 %{
4446 match(ConL);
4447
4448 op_cost(0);
4449 format %{ %}
4450 interface(CONST_INTER);
4451 %}
4452
4453 // 64 bit zero
4454 operand immL0()
4455 %{
4456 predicate(n->get_long() == 0);
4457 match(ConL);
4458
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 // 64 bit unit decrement
4465 operand immL_M1()
4466 %{
4467 predicate(n->get_long() == -1);
4468 match(ConL);
4469
4470 op_cost(0);
4471 format %{ %}
4472 interface(CONST_INTER);
4473 %}
4474
4475 // 64 bit integer valid for add sub immediate
4476 operand immLAddSub()
4477 %{
4478 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4479 match(ConL);
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 // 64 bit integer valid for addv subv immediate
4486 operand immLAddSubV()
4487 %{
4488 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4489 match(ConL);
4490
4491 op_cost(0);
4492 format %{ %}
4493 interface(CONST_INTER);
4494 %}
4495
4496 // 64 bit integer valid for logical immediate
4497 operand immLLog()
4498 %{
4499 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4500 match(ConL);
4501 op_cost(0);
4502 format %{ %}
4503 interface(CONST_INTER);
4504 %}
4505
4506 // Long Immediate: low 32-bit mask
4507 operand immL_32bits()
4508 %{
4509 predicate(n->get_long() == 0xFFFFFFFFL);
4510 match(ConL);
4511 op_cost(0);
4512 format %{ %}
4513 interface(CONST_INTER);
4514 %}
4515
4516 // Pointer operands
4517 // Pointer Immediate
4518 operand immP()
4519 %{
4520 match(ConP);
4521
4522 op_cost(0);
4523 format %{ %}
4524 interface(CONST_INTER);
4525 %}
4526
4527 // nullptr Pointer Immediate
4528 operand immP0()
4529 %{
4530 predicate(n->get_ptr() == 0);
4531 match(ConP);
4532
4533 op_cost(0);
4534 format %{ %}
4535 interface(CONST_INTER);
4536 %}
4537
4538 // Pointer Immediate One
4539 // this is used in object initialization (initial object header)
4540 operand immP_1()
4541 %{
4542 predicate(n->get_ptr() == 1);
4543 match(ConP);
4544
4545 op_cost(0);
4546 format %{ %}
4547 interface(CONST_INTER);
4548 %}
4549
4550 // AOT Runtime Constants Address
4551 operand immAOTRuntimeConstantsAddress()
4552 %{
4553 // Check if the address is in the range of AOT Runtime Constants
4554 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4555 match(ConP);
4556
4557 op_cost(0);
4558 format %{ %}
4559 interface(CONST_INTER);
4560 %}
4561
4562 // Float and Double operands
4563 // Double Immediate
4564 operand immD()
4565 %{
4566 match(ConD);
4567 op_cost(0);
4568 format %{ %}
4569 interface(CONST_INTER);
4570 %}
4571
4572 // Double Immediate: +0.0d
4573 operand immD0()
4574 %{
4575 predicate(jlong_cast(n->getd()) == 0);
4576 match(ConD);
4577
4578 op_cost(0);
4579 format %{ %}
4580 interface(CONST_INTER);
4581 %}
4582
4583 // constant 'double +0.0'.
4584 operand immDPacked()
4585 %{
4586 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4587 match(ConD);
4588 op_cost(0);
4589 format %{ %}
4590 interface(CONST_INTER);
4591 %}
4592
4593 // Float Immediate
4594 operand immF()
4595 %{
4596 match(ConF);
4597 op_cost(0);
4598 format %{ %}
4599 interface(CONST_INTER);
4600 %}
4601
4602 // Float Immediate: +0.0f.
4603 operand immF0()
4604 %{
4605 predicate(jint_cast(n->getf()) == 0);
4606 match(ConF);
4607
4608 op_cost(0);
4609 format %{ %}
4610 interface(CONST_INTER);
4611 %}
4612
4613 // Half Float (FP16) Immediate
4614 operand immH()
4615 %{
4616 match(ConH);
4617 op_cost(0);
4618 format %{ %}
4619 interface(CONST_INTER);
4620 %}
4621
4622 //
4623 operand immFPacked()
4624 %{
4625 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4626 match(ConF);
4627 op_cost(0);
4628 format %{ %}
4629 interface(CONST_INTER);
4630 %}
4631
4632 // Narrow pointer operands
4633 // Narrow Pointer Immediate
4634 operand immN()
4635 %{
4636 match(ConN);
4637
4638 op_cost(0);
4639 format %{ %}
4640 interface(CONST_INTER);
4641 %}
4642
4643 // Narrow nullptr Pointer Immediate
4644 operand immN0()
4645 %{
4646 predicate(n->get_narrowcon() == 0);
4647 match(ConN);
4648
4649 op_cost(0);
4650 format %{ %}
4651 interface(CONST_INTER);
4652 %}
4653
4654 operand immNKlass()
4655 %{
4656 match(ConNKlass);
4657
4658 op_cost(0);
4659 format %{ %}
4660 interface(CONST_INTER);
4661 %}
4662
4663 // Integer 32 bit Register Operands
4664 // Integer 32 bitRegister (excludes SP)
4665 operand iRegI()
4666 %{
4667 constraint(ALLOC_IN_RC(any_reg32));
4668 match(RegI);
4669 match(iRegINoSp);
4670 op_cost(0);
4671 format %{ %}
4672 interface(REG_INTER);
4673 %}
4674
4675 // Integer 32 bit Register not Special
4676 operand iRegINoSp()
4677 %{
4678 constraint(ALLOC_IN_RC(no_special_reg32));
4679 match(RegI);
4680 op_cost(0);
4681 format %{ %}
4682 interface(REG_INTER);
4683 %}
4684
4685 // Integer 64 bit Register Operands
4686 // Integer 64 bit Register (includes SP)
4687 operand iRegL()
4688 %{
4689 constraint(ALLOC_IN_RC(any_reg));
4690 match(RegL);
4691 match(iRegLNoSp);
4692 op_cost(0);
4693 format %{ %}
4694 interface(REG_INTER);
4695 %}
4696
4697 // Integer 64 bit Register not Special
4698 operand iRegLNoSp()
4699 %{
4700 constraint(ALLOC_IN_RC(no_special_reg));
4701 match(RegL);
4702 match(iRegL_R0);
4703 format %{ %}
4704 interface(REG_INTER);
4705 %}
4706
4707 // Pointer Register Operands
4708 // Pointer Register
4709 operand iRegP()
4710 %{
4711 constraint(ALLOC_IN_RC(ptr_reg));
4712 match(RegP);
4713 match(iRegPNoSp);
4714 match(iRegP_R0);
4715 //match(iRegP_R2);
4716 //match(iRegP_R4);
4717 match(iRegP_R5);
4718 match(thread_RegP);
4719 op_cost(0);
4720 format %{ %}
4721 interface(REG_INTER);
4722 %}
4723
4724 // Pointer 64 bit Register not Special
4725 operand iRegPNoSp()
4726 %{
4727 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4728 match(RegP);
4729 // match(iRegP);
4730 // match(iRegP_R0);
4731 // match(iRegP_R2);
4732 // match(iRegP_R4);
4733 // match(iRegP_R5);
4734 // match(thread_RegP);
4735 op_cost(0);
4736 format %{ %}
4737 interface(REG_INTER);
4738 %}
4739
4740 // This operand is not allowed to use rfp even if
4741 // rfp is not used to hold the frame pointer.
4742 operand iRegPNoSpNoRfp()
4743 %{
4744 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4745 match(RegP);
4746 match(iRegPNoSp);
4747 op_cost(0);
4748 format %{ %}
4749 interface(REG_INTER);
4750 %}
4751
4752 // Pointer 64 bit Register R0 only
4753 operand iRegP_R0()
4754 %{
4755 constraint(ALLOC_IN_RC(r0_reg));
4756 match(RegP);
4757 // match(iRegP);
4758 match(iRegPNoSp);
4759 op_cost(0);
4760 format %{ %}
4761 interface(REG_INTER);
4762 %}
4763
4764 // Pointer 64 bit Register R1 only
4765 operand iRegP_R1()
4766 %{
4767 constraint(ALLOC_IN_RC(r1_reg));
4768 match(RegP);
4769 // match(iRegP);
4770 match(iRegPNoSp);
4771 op_cost(0);
4772 format %{ %}
4773 interface(REG_INTER);
4774 %}
4775
4776 // Pointer 64 bit Register R2 only
4777 operand iRegP_R2()
4778 %{
4779 constraint(ALLOC_IN_RC(r2_reg));
4780 match(RegP);
4781 // match(iRegP);
4782 match(iRegPNoSp);
4783 op_cost(0);
4784 format %{ %}
4785 interface(REG_INTER);
4786 %}
4787
4788 // Pointer 64 bit Register R3 only
4789 operand iRegP_R3()
4790 %{
4791 constraint(ALLOC_IN_RC(r3_reg));
4792 match(RegP);
4793 // match(iRegP);
4794 match(iRegPNoSp);
4795 op_cost(0);
4796 format %{ %}
4797 interface(REG_INTER);
4798 %}
4799
4800 // Pointer 64 bit Register R4 only
4801 operand iRegP_R4()
4802 %{
4803 constraint(ALLOC_IN_RC(r4_reg));
4804 match(RegP);
4805 // match(iRegP);
4806 match(iRegPNoSp);
4807 op_cost(0);
4808 format %{ %}
4809 interface(REG_INTER);
4810 %}
4811
4812 // Pointer 64 bit Register R5 only
4813 operand iRegP_R5()
4814 %{
4815 constraint(ALLOC_IN_RC(r5_reg));
4816 match(RegP);
4817 // match(iRegP);
4818 match(iRegPNoSp);
4819 op_cost(0);
4820 format %{ %}
4821 interface(REG_INTER);
4822 %}
4823
4824 // Pointer 64 bit Register R10 only
4825 operand iRegP_R10()
4826 %{
4827 constraint(ALLOC_IN_RC(r10_reg));
4828 match(RegP);
4829 // match(iRegP);
4830 match(iRegPNoSp);
4831 op_cost(0);
4832 format %{ %}
4833 interface(REG_INTER);
4834 %}
4835
4836 // Long 64 bit Register R0 only
4837 operand iRegL_R0()
4838 %{
4839 constraint(ALLOC_IN_RC(r0_reg));
4840 match(RegL);
4841 match(iRegLNoSp);
4842 op_cost(0);
4843 format %{ %}
4844 interface(REG_INTER);
4845 %}
4846
4847 // Long 64 bit Register R11 only
4848 operand iRegL_R11()
4849 %{
4850 constraint(ALLOC_IN_RC(r11_reg));
4851 match(RegL);
4852 match(iRegLNoSp);
4853 op_cost(0);
4854 format %{ %}
4855 interface(REG_INTER);
4856 %}
4857
4858 // Register R0 only
4859 operand iRegI_R0()
4860 %{
4861 constraint(ALLOC_IN_RC(int_r0_reg));
4862 match(RegI);
4863 match(iRegINoSp);
4864 op_cost(0);
4865 format %{ %}
4866 interface(REG_INTER);
4867 %}
4868
4869 // Register R2 only
4870 operand iRegI_R2()
4871 %{
4872 constraint(ALLOC_IN_RC(int_r2_reg));
4873 match(RegI);
4874 match(iRegINoSp);
4875 op_cost(0);
4876 format %{ %}
4877 interface(REG_INTER);
4878 %}
4879
4880 // Register R3 only
4881 operand iRegI_R3()
4882 %{
4883 constraint(ALLOC_IN_RC(int_r3_reg));
4884 match(RegI);
4885 match(iRegINoSp);
4886 op_cost(0);
4887 format %{ %}
4888 interface(REG_INTER);
4889 %}
4890
4891
4892 // Register R4 only
4893 operand iRegI_R4()
4894 %{
4895 constraint(ALLOC_IN_RC(int_r4_reg));
4896 match(RegI);
4897 match(iRegINoSp);
4898 op_cost(0);
4899 format %{ %}
4900 interface(REG_INTER);
4901 %}
4902
4903
4904 // Pointer Register Operands
4905 // Narrow Pointer Register
4906 operand iRegN()
4907 %{
4908 constraint(ALLOC_IN_RC(any_reg32));
4909 match(RegN);
4910 match(iRegNNoSp);
4911 op_cost(0);
4912 format %{ %}
4913 interface(REG_INTER);
4914 %}
4915
4916 // Integer 64 bit Register not Special
4917 operand iRegNNoSp()
4918 %{
4919 constraint(ALLOC_IN_RC(no_special_reg32));
4920 match(RegN);
4921 op_cost(0);
4922 format %{ %}
4923 interface(REG_INTER);
4924 %}
4925
4926 // Float Register
4927 // Float register operands
4928 operand vRegF()
4929 %{
4930 constraint(ALLOC_IN_RC(float_reg));
4931 match(RegF);
4932
4933 op_cost(0);
4934 format %{ %}
4935 interface(REG_INTER);
4936 %}
4937
4938 // Double Register
4939 // Double register operands
4940 operand vRegD()
4941 %{
4942 constraint(ALLOC_IN_RC(double_reg));
4943 match(RegD);
4944
4945 op_cost(0);
4946 format %{ %}
4947 interface(REG_INTER);
4948 %}
4949
4950 // Generic vector class. This will be used for
4951 // all vector operands, including NEON and SVE.
4952 operand vReg()
4953 %{
4954 constraint(ALLOC_IN_RC(dynamic));
4955 match(VecA);
4956 match(VecD);
4957 match(VecX);
4958
4959 op_cost(0);
4960 format %{ %}
4961 interface(REG_INTER);
4962 %}
4963
4964 operand vReg_V10()
4965 %{
4966 constraint(ALLOC_IN_RC(v10_veca_reg));
4967 match(vReg);
4968
4969 op_cost(0);
4970 format %{ %}
4971 interface(REG_INTER);
4972 %}
4973
4974 operand vReg_V11()
4975 %{
4976 constraint(ALLOC_IN_RC(v11_veca_reg));
4977 match(vReg);
4978
4979 op_cost(0);
4980 format %{ %}
4981 interface(REG_INTER);
4982 %}
4983
4984 operand vReg_V12()
4985 %{
4986 constraint(ALLOC_IN_RC(v12_veca_reg));
4987 match(vReg);
4988
4989 op_cost(0);
4990 format %{ %}
4991 interface(REG_INTER);
4992 %}
4993
4994 operand vReg_V13()
4995 %{
4996 constraint(ALLOC_IN_RC(v13_veca_reg));
4997 match(vReg);
4998
4999 op_cost(0);
5000 format %{ %}
5001 interface(REG_INTER);
5002 %}
5003
5004 operand vReg_V17()
5005 %{
5006 constraint(ALLOC_IN_RC(v17_veca_reg));
5007 match(vReg);
5008
5009 op_cost(0);
5010 format %{ %}
5011 interface(REG_INTER);
5012 %}
5013
5014 operand vReg_V18()
5015 %{
5016 constraint(ALLOC_IN_RC(v18_veca_reg));
5017 match(vReg);
5018
5019 op_cost(0);
5020 format %{ %}
5021 interface(REG_INTER);
5022 %}
5023
5024 operand vReg_V23()
5025 %{
5026 constraint(ALLOC_IN_RC(v23_veca_reg));
5027 match(vReg);
5028
5029 op_cost(0);
5030 format %{ %}
5031 interface(REG_INTER);
5032 %}
5033
5034 operand vReg_V24()
5035 %{
5036 constraint(ALLOC_IN_RC(v24_veca_reg));
5037 match(vReg);
5038
5039 op_cost(0);
5040 format %{ %}
5041 interface(REG_INTER);
5042 %}
5043
5044 operand vecA()
5045 %{
5046 constraint(ALLOC_IN_RC(vectora_reg));
5047 match(VecA);
5048
5049 op_cost(0);
5050 format %{ %}
5051 interface(REG_INTER);
5052 %}
5053
5054 operand vecD()
5055 %{
5056 constraint(ALLOC_IN_RC(vectord_reg));
5057 match(VecD);
5058
5059 op_cost(0);
5060 format %{ %}
5061 interface(REG_INTER);
5062 %}
5063
5064 operand vecX()
5065 %{
5066 constraint(ALLOC_IN_RC(vectorx_reg));
5067 match(VecX);
5068
5069 op_cost(0);
5070 format %{ %}
5071 interface(REG_INTER);
5072 %}
5073
5074 operand vRegD_V0()
5075 %{
5076 constraint(ALLOC_IN_RC(v0_reg));
5077 match(RegD);
5078 op_cost(0);
5079 format %{ %}
5080 interface(REG_INTER);
5081 %}
5082
5083 operand vRegD_V1()
5084 %{
5085 constraint(ALLOC_IN_RC(v1_reg));
5086 match(RegD);
5087 op_cost(0);
5088 format %{ %}
5089 interface(REG_INTER);
5090 %}
5091
5092 operand vRegD_V2()
5093 %{
5094 constraint(ALLOC_IN_RC(v2_reg));
5095 match(RegD);
5096 op_cost(0);
5097 format %{ %}
5098 interface(REG_INTER);
5099 %}
5100
5101 operand vRegD_V3()
5102 %{
5103 constraint(ALLOC_IN_RC(v3_reg));
5104 match(RegD);
5105 op_cost(0);
5106 format %{ %}
5107 interface(REG_INTER);
5108 %}
5109
5110 operand vRegD_V4()
5111 %{
5112 constraint(ALLOC_IN_RC(v4_reg));
5113 match(RegD);
5114 op_cost(0);
5115 format %{ %}
5116 interface(REG_INTER);
5117 %}
5118
5119 operand vRegD_V5()
5120 %{
5121 constraint(ALLOC_IN_RC(v5_reg));
5122 match(RegD);
5123 op_cost(0);
5124 format %{ %}
5125 interface(REG_INTER);
5126 %}
5127
5128 operand vRegD_V6()
5129 %{
5130 constraint(ALLOC_IN_RC(v6_reg));
5131 match(RegD);
5132 op_cost(0);
5133 format %{ %}
5134 interface(REG_INTER);
5135 %}
5136
5137 operand vRegD_V7()
5138 %{
5139 constraint(ALLOC_IN_RC(v7_reg));
5140 match(RegD);
5141 op_cost(0);
5142 format %{ %}
5143 interface(REG_INTER);
5144 %}
5145
5146 operand vRegD_V12()
5147 %{
5148 constraint(ALLOC_IN_RC(v12_reg));
5149 match(RegD);
5150 op_cost(0);
5151 format %{ %}
5152 interface(REG_INTER);
5153 %}
5154
5155 operand vRegD_V13()
5156 %{
5157 constraint(ALLOC_IN_RC(v13_reg));
5158 match(RegD);
5159 op_cost(0);
5160 format %{ %}
5161 interface(REG_INTER);
5162 %}
5163
5164 operand pReg()
5165 %{
5166 constraint(ALLOC_IN_RC(pr_reg));
5167 match(RegVectMask);
5168 match(pRegGov);
5169 op_cost(0);
5170 format %{ %}
5171 interface(REG_INTER);
5172 %}
5173
5174 operand pRegGov()
5175 %{
5176 constraint(ALLOC_IN_RC(gov_pr));
5177 match(RegVectMask);
5178 match(pReg);
5179 op_cost(0);
5180 format %{ %}
5181 interface(REG_INTER);
5182 %}
5183
5184 operand pRegGov_P0()
5185 %{
5186 constraint(ALLOC_IN_RC(p0_reg));
5187 match(RegVectMask);
5188 op_cost(0);
5189 format %{ %}
5190 interface(REG_INTER);
5191 %}
5192
5193 operand pRegGov_P1()
5194 %{
5195 constraint(ALLOC_IN_RC(p1_reg));
5196 match(RegVectMask);
5197 op_cost(0);
5198 format %{ %}
5199 interface(REG_INTER);
5200 %}
5201
5202 // Flags register, used as output of signed compare instructions
5203
5204 // note that on AArch64 we also use this register as the output for
5205 // for floating point compare instructions (CmpF CmpD). this ensures
5206 // that ordered inequality tests use GT, GE, LT or LE none of which
5207 // pass through cases where the result is unordered i.e. one or both
5208 // inputs to the compare is a NaN. this means that the ideal code can
5209 // replace e.g. a GT with an LE and not end up capturing the NaN case
5210 // (where the comparison should always fail). EQ and NE tests are
5211 // always generated in ideal code so that unordered folds into the NE
5212 // case, matching the behaviour of AArch64 NE.
5213 //
5214 // This differs from x86 where the outputs of FP compares use a
5215 // special FP flags registers and where compares based on this
5216 // register are distinguished into ordered inequalities (cmpOpUCF) and
5217 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5218 // to explicitly handle the unordered case in branches. x86 also has
5219 // to include extra CMoveX rules to accept a cmpOpUCF input.
5220
5221 operand rFlagsReg()
5222 %{
5223 constraint(ALLOC_IN_RC(int_flags));
5224 match(RegFlags);
5225
5226 op_cost(0);
5227 format %{ "RFLAGS" %}
5228 interface(REG_INTER);
5229 %}
5230
5231 // Flags register, used as output of unsigned compare instructions
5232 operand rFlagsRegU()
5233 %{
5234 constraint(ALLOC_IN_RC(int_flags));
5235 match(RegFlags);
5236
5237 op_cost(0);
5238 format %{ "RFLAGSU" %}
5239 interface(REG_INTER);
5240 %}
5241
5242 // Special Registers
5243
5244 // Method Register
5245 operand inline_cache_RegP(iRegP reg)
5246 %{
5247 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5248 match(reg);
5249 match(iRegPNoSp);
5250 op_cost(0);
5251 format %{ %}
5252 interface(REG_INTER);
5253 %}
5254
5255 // Thread Register
5256 operand thread_RegP(iRegP reg)
5257 %{
5258 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5259 match(reg);
5260 op_cost(0);
5261 format %{ %}
5262 interface(REG_INTER);
5263 %}
5264
5265 //----------Memory Operands----------------------------------------------------
5266
5267 operand indirect(iRegP reg)
5268 %{
5269 constraint(ALLOC_IN_RC(ptr_reg));
5270 match(reg);
5271 op_cost(0);
5272 format %{ "[$reg]" %}
5273 interface(MEMORY_INTER) %{
5274 base($reg);
5275 index(0xffffffff);
5276 scale(0x0);
5277 disp(0x0);
5278 %}
5279 %}
5280
5281 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5282 %{
5283 constraint(ALLOC_IN_RC(ptr_reg));
5284 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5285 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5286 op_cost(0);
5287 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5288 interface(MEMORY_INTER) %{
5289 base($reg);
5290 index($ireg);
5291 scale($scale);
5292 disp(0x0);
5293 %}
5294 %}
5295
5296 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5297 %{
5298 constraint(ALLOC_IN_RC(ptr_reg));
5299 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5300 match(AddP reg (LShiftL lreg scale));
5301 op_cost(0);
5302 format %{ "$reg, $lreg lsl($scale)" %}
5303 interface(MEMORY_INTER) %{
5304 base($reg);
5305 index($lreg);
5306 scale($scale);
5307 disp(0x0);
5308 %}
5309 %}
5310
5311 operand indIndexI2L(iRegP reg, iRegI ireg)
5312 %{
5313 constraint(ALLOC_IN_RC(ptr_reg));
5314 match(AddP reg (ConvI2L ireg));
5315 op_cost(0);
5316 format %{ "$reg, $ireg, 0, I2L" %}
5317 interface(MEMORY_INTER) %{
5318 base($reg);
5319 index($ireg);
5320 scale(0x0);
5321 disp(0x0);
5322 %}
5323 %}
5324
5325 operand indIndex(iRegP reg, iRegL lreg)
5326 %{
5327 constraint(ALLOC_IN_RC(ptr_reg));
5328 match(AddP reg lreg);
5329 op_cost(0);
5330 format %{ "$reg, $lreg" %}
5331 interface(MEMORY_INTER) %{
5332 base($reg);
5333 index($lreg);
5334 scale(0x0);
5335 disp(0x0);
5336 %}
5337 %}
5338
5339 operand indOffI1(iRegP reg, immIOffset1 off)
5340 %{
5341 constraint(ALLOC_IN_RC(ptr_reg));
5342 match(AddP reg off);
5343 op_cost(0);
5344 format %{ "[$reg, $off]" %}
5345 interface(MEMORY_INTER) %{
5346 base($reg);
5347 index(0xffffffff);
5348 scale(0x0);
5349 disp($off);
5350 %}
5351 %}
5352
5353 operand indOffI2(iRegP reg, immIOffset2 off)
5354 %{
5355 constraint(ALLOC_IN_RC(ptr_reg));
5356 match(AddP reg off);
5357 op_cost(0);
5358 format %{ "[$reg, $off]" %}
5359 interface(MEMORY_INTER) %{
5360 base($reg);
5361 index(0xffffffff);
5362 scale(0x0);
5363 disp($off);
5364 %}
5365 %}
5366
5367 operand indOffI4(iRegP reg, immIOffset4 off)
5368 %{
5369 constraint(ALLOC_IN_RC(ptr_reg));
5370 match(AddP reg off);
5371 op_cost(0);
5372 format %{ "[$reg, $off]" %}
5373 interface(MEMORY_INTER) %{
5374 base($reg);
5375 index(0xffffffff);
5376 scale(0x0);
5377 disp($off);
5378 %}
5379 %}
5380
5381 operand indOffI8(iRegP reg, immIOffset8 off)
5382 %{
5383 constraint(ALLOC_IN_RC(ptr_reg));
5384 match(AddP reg off);
5385 op_cost(0);
5386 format %{ "[$reg, $off]" %}
5387 interface(MEMORY_INTER) %{
5388 base($reg);
5389 index(0xffffffff);
5390 scale(0x0);
5391 disp($off);
5392 %}
5393 %}
5394
5395 operand indOffI16(iRegP reg, immIOffset16 off)
5396 %{
5397 constraint(ALLOC_IN_RC(ptr_reg));
5398 match(AddP reg off);
5399 op_cost(0);
5400 format %{ "[$reg, $off]" %}
5401 interface(MEMORY_INTER) %{
5402 base($reg);
5403 index(0xffffffff);
5404 scale(0x0);
5405 disp($off);
5406 %}
5407 %}
5408
5409 operand indOffL1(iRegP reg, immLoffset1 off)
5410 %{
5411 constraint(ALLOC_IN_RC(ptr_reg));
5412 match(AddP reg off);
5413 op_cost(0);
5414 format %{ "[$reg, $off]" %}
5415 interface(MEMORY_INTER) %{
5416 base($reg);
5417 index(0xffffffff);
5418 scale(0x0);
5419 disp($off);
5420 %}
5421 %}
5422
5423 operand indOffL2(iRegP reg, immLoffset2 off)
5424 %{
5425 constraint(ALLOC_IN_RC(ptr_reg));
5426 match(AddP reg off);
5427 op_cost(0);
5428 format %{ "[$reg, $off]" %}
5429 interface(MEMORY_INTER) %{
5430 base($reg);
5431 index(0xffffffff);
5432 scale(0x0);
5433 disp($off);
5434 %}
5435 %}
5436
5437 operand indOffL4(iRegP reg, immLoffset4 off)
5438 %{
5439 constraint(ALLOC_IN_RC(ptr_reg));
5440 match(AddP reg off);
5441 op_cost(0);
5442 format %{ "[$reg, $off]" %}
5443 interface(MEMORY_INTER) %{
5444 base($reg);
5445 index(0xffffffff);
5446 scale(0x0);
5447 disp($off);
5448 %}
5449 %}
5450
5451 operand indOffL8(iRegP reg, immLoffset8 off)
5452 %{
5453 constraint(ALLOC_IN_RC(ptr_reg));
5454 match(AddP reg off);
5455 op_cost(0);
5456 format %{ "[$reg, $off]" %}
5457 interface(MEMORY_INTER) %{
5458 base($reg);
5459 index(0xffffffff);
5460 scale(0x0);
5461 disp($off);
5462 %}
5463 %}
5464
5465 operand indOffL16(iRegP reg, immLoffset16 off)
5466 %{
5467 constraint(ALLOC_IN_RC(ptr_reg));
5468 match(AddP reg off);
5469 op_cost(0);
5470 format %{ "[$reg, $off]" %}
5471 interface(MEMORY_INTER) %{
5472 base($reg);
5473 index(0xffffffff);
5474 scale(0x0);
5475 disp($off);
5476 %}
5477 %}
5478
5479 operand indirectX2P(iRegL reg)
5480 %{
5481 constraint(ALLOC_IN_RC(ptr_reg));
5482 match(CastX2P reg);
5483 op_cost(0);
5484 format %{ "[$reg]\t# long -> ptr" %}
5485 interface(MEMORY_INTER) %{
5486 base($reg);
5487 index(0xffffffff);
5488 scale(0x0);
5489 disp(0x0);
5490 %}
5491 %}
5492
5493 operand indOffX2P(iRegL reg, immLOffset off)
5494 %{
5495 constraint(ALLOC_IN_RC(ptr_reg));
5496 match(AddP (CastX2P reg) off);
5497 op_cost(0);
5498 format %{ "[$reg, $off]\t# long -> ptr" %}
5499 interface(MEMORY_INTER) %{
5500 base($reg);
5501 index(0xffffffff);
5502 scale(0x0);
5503 disp($off);
5504 %}
5505 %}
5506
5507 operand indirectN(iRegN reg)
5508 %{
5509 predicate(CompressedOops::shift() == 0);
5510 constraint(ALLOC_IN_RC(ptr_reg));
5511 match(DecodeN reg);
5512 op_cost(0);
5513 format %{ "[$reg]\t# narrow" %}
5514 interface(MEMORY_INTER) %{
5515 base($reg);
5516 index(0xffffffff);
5517 scale(0x0);
5518 disp(0x0);
5519 %}
5520 %}
5521
5522 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5523 %{
5524 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5525 constraint(ALLOC_IN_RC(ptr_reg));
5526 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5527 op_cost(0);
5528 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5529 interface(MEMORY_INTER) %{
5530 base($reg);
5531 index($ireg);
5532 scale($scale);
5533 disp(0x0);
5534 %}
5535 %}
5536
5537 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5538 %{
5539 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5540 constraint(ALLOC_IN_RC(ptr_reg));
5541 match(AddP (DecodeN reg) (LShiftL lreg scale));
5542 op_cost(0);
5543 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5544 interface(MEMORY_INTER) %{
5545 base($reg);
5546 index($lreg);
5547 scale($scale);
5548 disp(0x0);
5549 %}
5550 %}
5551
5552 operand indIndexI2LN(iRegN reg, iRegI ireg)
5553 %{
5554 predicate(CompressedOops::shift() == 0);
5555 constraint(ALLOC_IN_RC(ptr_reg));
5556 match(AddP (DecodeN reg) (ConvI2L ireg));
5557 op_cost(0);
5558 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5559 interface(MEMORY_INTER) %{
5560 base($reg);
5561 index($ireg);
5562 scale(0x0);
5563 disp(0x0);
5564 %}
5565 %}
5566
5567 operand indIndexN(iRegN reg, iRegL lreg)
5568 %{
5569 predicate(CompressedOops::shift() == 0);
5570 constraint(ALLOC_IN_RC(ptr_reg));
5571 match(AddP (DecodeN reg) lreg);
5572 op_cost(0);
5573 format %{ "$reg, $lreg\t# narrow" %}
5574 interface(MEMORY_INTER) %{
5575 base($reg);
5576 index($lreg);
5577 scale(0x0);
5578 disp(0x0);
5579 %}
5580 %}
5581
5582 operand indOffIN(iRegN reg, immIOffset off)
5583 %{
5584 predicate(CompressedOops::shift() == 0);
5585 constraint(ALLOC_IN_RC(ptr_reg));
5586 match(AddP (DecodeN reg) off);
5587 op_cost(0);
5588 format %{ "[$reg, $off]\t# narrow" %}
5589 interface(MEMORY_INTER) %{
5590 base($reg);
5591 index(0xffffffff);
5592 scale(0x0);
5593 disp($off);
5594 %}
5595 %}
5596
5597 operand indOffLN(iRegN reg, immLOffset off)
5598 %{
5599 predicate(CompressedOops::shift() == 0);
5600 constraint(ALLOC_IN_RC(ptr_reg));
5601 match(AddP (DecodeN reg) off);
5602 op_cost(0);
5603 format %{ "[$reg, $off]\t# narrow" %}
5604 interface(MEMORY_INTER) %{
5605 base($reg);
5606 index(0xffffffff);
5607 scale(0x0);
5608 disp($off);
5609 %}
5610 %}
5611
5612
5613 //----------Special Memory Operands--------------------------------------------
5614 // Stack Slot Operand - This operand is used for loading and storing temporary
5615 // values on the stack where a match requires a value to
5616 // flow through memory.
5617 operand stackSlotP(sRegP reg)
5618 %{
5619 constraint(ALLOC_IN_RC(stack_slots));
5620 op_cost(100);
5621 // No match rule because this operand is only generated in matching
5622 // match(RegP);
5623 format %{ "[$reg]" %}
5624 interface(MEMORY_INTER) %{
5625 base(0x1e); // RSP
5626 index(0x0); // No Index
5627 scale(0x0); // No Scale
5628 disp($reg); // Stack Offset
5629 %}
5630 %}
5631
5632 operand stackSlotI(sRegI reg)
5633 %{
5634 constraint(ALLOC_IN_RC(stack_slots));
5635 // No match rule because this operand is only generated in matching
5636 // match(RegI);
5637 format %{ "[$reg]" %}
5638 interface(MEMORY_INTER) %{
5639 base(0x1e); // RSP
5640 index(0x0); // No Index
5641 scale(0x0); // No Scale
5642 disp($reg); // Stack Offset
5643 %}
5644 %}
5645
5646 operand stackSlotF(sRegF reg)
5647 %{
5648 constraint(ALLOC_IN_RC(stack_slots));
5649 // No match rule because this operand is only generated in matching
5650 // match(RegF);
5651 format %{ "[$reg]" %}
5652 interface(MEMORY_INTER) %{
5653 base(0x1e); // RSP
5654 index(0x0); // No Index
5655 scale(0x0); // No Scale
5656 disp($reg); // Stack Offset
5657 %}
5658 %}
5659
5660 operand stackSlotD(sRegD reg)
5661 %{
5662 constraint(ALLOC_IN_RC(stack_slots));
5663 // No match rule because this operand is only generated in matching
5664 // match(RegD);
5665 format %{ "[$reg]" %}
5666 interface(MEMORY_INTER) %{
5667 base(0x1e); // RSP
5668 index(0x0); // No Index
5669 scale(0x0); // No Scale
5670 disp($reg); // Stack Offset
5671 %}
5672 %}
5673
5674 operand stackSlotL(sRegL reg)
5675 %{
5676 constraint(ALLOC_IN_RC(stack_slots));
5677 // No match rule because this operand is only generated in matching
5678 // match(RegL);
5679 format %{ "[$reg]" %}
5680 interface(MEMORY_INTER) %{
5681 base(0x1e); // RSP
5682 index(0x0); // No Index
5683 scale(0x0); // No Scale
5684 disp($reg); // Stack Offset
5685 %}
5686 %}
5687
5688 // Operands for expressing Control Flow
5689 // NOTE: Label is a predefined operand which should not be redefined in
5690 // the AD file. It is generically handled within the ADLC.
5691
5692 //----------Conditional Branch Operands----------------------------------------
5693 // Comparison Op - This is the operation of the comparison, and is limited to
5694 // the following set of codes:
5695 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5696 //
5697 // Other attributes of the comparison, such as unsignedness, are specified
5698 // by the comparison instruction that sets a condition code flags register.
5699 // That result is represented by a flags operand whose subtype is appropriate
5700 // to the unsignedness (etc.) of the comparison.
5701 //
5702 // Later, the instruction which matches both the Comparison Op (a Bool) and
5703 // the flags (produced by the Cmp) specifies the coding of the comparison op
5704 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5705
5706 // used for signed integral comparisons and fp comparisons
5707
5708 operand cmpOp()
5709 %{
5710 match(Bool);
5711
5712 format %{ "" %}
5713 interface(COND_INTER) %{
5714 equal(0x0, "eq");
5715 not_equal(0x1, "ne");
5716 less(0xb, "lt");
5717 greater_equal(0xa, "ge");
5718 less_equal(0xd, "le");
5719 greater(0xc, "gt");
5720 overflow(0x6, "vs");
5721 no_overflow(0x7, "vc");
5722 %}
5723 %}
5724
5725 // used for unsigned integral comparisons
5726
5727 operand cmpOpU()
5728 %{
5729 match(Bool);
5730
5731 format %{ "" %}
5732 interface(COND_INTER) %{
5733 equal(0x0, "eq");
5734 not_equal(0x1, "ne");
5735 less(0x3, "lo");
5736 greater_equal(0x2, "hs");
5737 less_equal(0x9, "ls");
5738 greater(0x8, "hi");
5739 overflow(0x6, "vs");
5740 no_overflow(0x7, "vc");
5741 %}
5742 %}
5743
5744 // used for certain integral comparisons which can be
5745 // converted to cbxx or tbxx instructions
5746
5747 operand cmpOpEqNe()
5748 %{
5749 match(Bool);
5750 op_cost(0);
5751 predicate(n->as_Bool()->_test._test == BoolTest::ne
5752 || n->as_Bool()->_test._test == BoolTest::eq);
5753
5754 format %{ "" %}
5755 interface(COND_INTER) %{
5756 equal(0x0, "eq");
5757 not_equal(0x1, "ne");
5758 less(0xb, "lt");
5759 greater_equal(0xa, "ge");
5760 less_equal(0xd, "le");
5761 greater(0xc, "gt");
5762 overflow(0x6, "vs");
5763 no_overflow(0x7, "vc");
5764 %}
5765 %}
5766
5767 // used for certain integral comparisons which can be
5768 // converted to cbxx or tbxx instructions
5769
5770 operand cmpOpLtGe()
5771 %{
5772 match(Bool);
5773 op_cost(0);
5774
5775 predicate(n->as_Bool()->_test._test == BoolTest::lt
5776 || n->as_Bool()->_test._test == BoolTest::ge);
5777
5778 format %{ "" %}
5779 interface(COND_INTER) %{
5780 equal(0x0, "eq");
5781 not_equal(0x1, "ne");
5782 less(0xb, "lt");
5783 greater_equal(0xa, "ge");
5784 less_equal(0xd, "le");
5785 greater(0xc, "gt");
5786 overflow(0x6, "vs");
5787 no_overflow(0x7, "vc");
5788 %}
5789 %}
5790
5791 // used for certain unsigned integral comparisons which can be
5792 // converted to cbxx or tbxx instructions
5793
5794 operand cmpOpUEqNeLeGt()
5795 %{
5796 match(Bool);
5797 op_cost(0);
5798
5799 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5800 n->as_Bool()->_test._test == BoolTest::ne ||
5801 n->as_Bool()->_test._test == BoolTest::le ||
5802 n->as_Bool()->_test._test == BoolTest::gt);
5803
5804 format %{ "" %}
5805 interface(COND_INTER) %{
5806 equal(0x0, "eq");
5807 not_equal(0x1, "ne");
5808 less(0x3, "lo");
5809 greater_equal(0x2, "hs");
5810 less_equal(0x9, "ls");
5811 greater(0x8, "hi");
5812 overflow(0x6, "vs");
5813 no_overflow(0x7, "vc");
5814 %}
5815 %}
5816
5817 // Special operand allowing long args to int ops to be truncated for free
5818
5819 operand iRegL2I(iRegL reg) %{
5820
5821 op_cost(0);
5822
5823 match(ConvL2I reg);
5824
5825 format %{ "l2i($reg)" %}
5826
5827 interface(REG_INTER)
5828 %}
5829
5830 operand iRegL2P(iRegL reg) %{
5831
5832 op_cost(0);
5833
5834 match(CastX2P reg);
5835
5836 format %{ "l2p($reg)" %}
5837
5838 interface(REG_INTER)
5839 %}
5840
5841 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5842 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5843 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5844 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5845
5846 //----------OPERAND CLASSES----------------------------------------------------
5847 // Operand Classes are groups of operands that are used as to simplify
5848 // instruction definitions by not requiring the AD writer to specify
5849 // separate instructions for every form of operand when the
5850 // instruction accepts multiple operand types with the same basic
5851 // encoding and format. The classic case of this is memory operands.
5852
5853 // memory is used to define read/write location for load/store
5854 // instruction defs. we can turn a memory op into an Address
5855
5856 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5857 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5858
5859 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5860 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5861
5862 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5863 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5864
5865 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5866 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5867
5868 // All of the memory operands. For the pipeline description.
5869 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5870 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5871 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5872
5873
5874 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5875 // operations. it allows the src to be either an iRegI or a (ConvL2I
5876 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5877 // can be elided because the 32-bit instruction will just employ the
5878 // lower 32 bits anyway.
5879 //
5880 // n.b. this does not elide all L2I conversions. if the truncated
5881 // value is consumed by more than one operation then the ConvL2I
5882 // cannot be bundled into the consuming nodes so an l2i gets planted
5883 // (actually a movw $dst $src) and the downstream instructions consume
5884 // the result of the l2i as an iRegI input. That's a shame since the
5885 // movw is actually redundant but its not too costly.
5886
5887 opclass iRegIorL2I(iRegI, iRegL2I);
5888 opclass iRegPorL2P(iRegP, iRegL2P);
5889
5890 //----------PIPELINE-----------------------------------------------------------
5891 // Rules which define the behavior of the target architectures pipeline.
5892
5893 // For specific pipelines, eg A53, define the stages of that pipeline
5894 //pipe_desc(ISS, EX1, EX2, WR);
5895 #define ISS S0
5896 #define EX1 S1
5897 #define EX2 S2
5898 #define WR S3
5899
5900 // Integer ALU reg operation
5901 pipeline %{
5902
5903 attributes %{
5904 // ARM instructions are of fixed length
5905 fixed_size_instructions; // Fixed size instructions TODO does
5906 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5907 // ARM instructions come in 32-bit word units
5908 instruction_unit_size = 4; // An instruction is 4 bytes long
5909 instruction_fetch_unit_size = 64; // The processor fetches one line
5910 instruction_fetch_units = 1; // of 64 bytes
5911 %}
5912
5913 // We don't use an actual pipeline model so don't care about resources
5914 // or description. we do use pipeline classes to introduce fixed
5915 // latencies
5916
5917 //----------RESOURCES----------------------------------------------------------
5918 // Resources are the functional units available to the machine
5919
5920 resources( INS0, INS1, INS01 = INS0 | INS1,
5921 ALU0, ALU1, ALU = ALU0 | ALU1,
5922 MAC,
5923 DIV,
5924 BRANCH,
5925 LDST,
5926 NEON_FP);
5927
5928 //----------PIPELINE DESCRIPTION-----------------------------------------------
5929 // Pipeline Description specifies the stages in the machine's pipeline
5930
5931 // Define the pipeline as a generic 6 stage pipeline
5932 pipe_desc(S0, S1, S2, S3, S4, S5);
5933
5934 //----------PIPELINE CLASSES---------------------------------------------------
5935 // Pipeline Classes describe the stages in which input and output are
5936 // referenced by the hardware pipeline.
5937
5938 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5939 %{
5940 single_instruction;
5941 src1 : S1(read);
5942 src2 : S2(read);
5943 dst : S5(write);
5944 INS01 : ISS;
5945 NEON_FP : S5;
5946 %}
5947
5948 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5949 %{
5950 single_instruction;
5951 src1 : S1(read);
5952 src2 : S2(read);
5953 dst : S5(write);
5954 INS01 : ISS;
5955 NEON_FP : S5;
5956 %}
5957
5958 pipe_class fp_uop_s(vRegF dst, vRegF src)
5959 %{
5960 single_instruction;
5961 src : S1(read);
5962 dst : S5(write);
5963 INS01 : ISS;
5964 NEON_FP : S5;
5965 %}
5966
5967 pipe_class fp_uop_d(vRegD dst, vRegD src)
5968 %{
5969 single_instruction;
5970 src : S1(read);
5971 dst : S5(write);
5972 INS01 : ISS;
5973 NEON_FP : S5;
5974 %}
5975
5976 pipe_class fp_d2f(vRegF dst, vRegD src)
5977 %{
5978 single_instruction;
5979 src : S1(read);
5980 dst : S5(write);
5981 INS01 : ISS;
5982 NEON_FP : S5;
5983 %}
5984
5985 pipe_class fp_f2d(vRegD dst, vRegF src)
5986 %{
5987 single_instruction;
5988 src : S1(read);
5989 dst : S5(write);
5990 INS01 : ISS;
5991 NEON_FP : S5;
5992 %}
5993
5994 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5995 %{
5996 single_instruction;
5997 src : S1(read);
5998 dst : S5(write);
5999 INS01 : ISS;
6000 NEON_FP : S5;
6001 %}
6002
6003 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6004 %{
6005 single_instruction;
6006 src : S1(read);
6007 dst : S5(write);
6008 INS01 : ISS;
6009 NEON_FP : S5;
6010 %}
6011
6012 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6013 %{
6014 single_instruction;
6015 src : S1(read);
6016 dst : S5(write);
6017 INS01 : ISS;
6018 NEON_FP : S5;
6019 %}
6020
6021 pipe_class fp_l2f(vRegF dst, iRegL src)
6022 %{
6023 single_instruction;
6024 src : S1(read);
6025 dst : S5(write);
6026 INS01 : ISS;
6027 NEON_FP : S5;
6028 %}
6029
6030 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6031 %{
6032 single_instruction;
6033 src : S1(read);
6034 dst : S5(write);
6035 INS01 : ISS;
6036 NEON_FP : S5;
6037 %}
6038
6039 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6040 %{
6041 single_instruction;
6042 src : S1(read);
6043 dst : S5(write);
6044 INS01 : ISS;
6045 NEON_FP : S5;
6046 %}
6047
6048 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6049 %{
6050 single_instruction;
6051 src : S1(read);
6052 dst : S5(write);
6053 INS01 : ISS;
6054 NEON_FP : S5;
6055 %}
6056
6057 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6058 %{
6059 single_instruction;
6060 src : S1(read);
6061 dst : S5(write);
6062 INS01 : ISS;
6063 NEON_FP : S5;
6064 %}
6065
6066 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6067 %{
6068 single_instruction;
6069 src1 : S1(read);
6070 src2 : S2(read);
6071 dst : S5(write);
6072 INS0 : ISS;
6073 NEON_FP : S5;
6074 %}
6075
6076 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6077 %{
6078 single_instruction;
6079 src1 : S1(read);
6080 src2 : S2(read);
6081 dst : S5(write);
6082 INS0 : ISS;
6083 NEON_FP : S5;
6084 %}
6085
6086 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6087 %{
6088 single_instruction;
6089 cr : S1(read);
6090 src1 : S1(read);
6091 src2 : S1(read);
6092 dst : S3(write);
6093 INS01 : ISS;
6094 NEON_FP : S3;
6095 %}
6096
6097 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6098 %{
6099 single_instruction;
6100 cr : S1(read);
6101 src1 : S1(read);
6102 src2 : S1(read);
6103 dst : S3(write);
6104 INS01 : ISS;
6105 NEON_FP : S3;
6106 %}
6107
6108 pipe_class fp_imm_s(vRegF dst)
6109 %{
6110 single_instruction;
6111 dst : S3(write);
6112 INS01 : ISS;
6113 NEON_FP : S3;
6114 %}
6115
6116 pipe_class fp_imm_d(vRegD dst)
6117 %{
6118 single_instruction;
6119 dst : S3(write);
6120 INS01 : ISS;
6121 NEON_FP : S3;
6122 %}
6123
6124 pipe_class fp_load_constant_s(vRegF dst)
6125 %{
6126 single_instruction;
6127 dst : S4(write);
6128 INS01 : ISS;
6129 NEON_FP : S4;
6130 %}
6131
6132 pipe_class fp_load_constant_d(vRegD dst)
6133 %{
6134 single_instruction;
6135 dst : S4(write);
6136 INS01 : ISS;
6137 NEON_FP : S4;
6138 %}
6139
6140 //------- Integer ALU operations --------------------------
6141
6142 // Integer ALU reg-reg operation
6143 // Operands needed in EX1, result generated in EX2
6144 // Eg. ADD x0, x1, x2
6145 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6146 %{
6147 single_instruction;
6148 dst : EX2(write);
6149 src1 : EX1(read);
6150 src2 : EX1(read);
6151 INS01 : ISS; // Dual issue as instruction 0 or 1
6152 ALU : EX2;
6153 %}
6154
6155 // Integer ALU reg-reg operation with constant shift
6156 // Shifted register must be available in LATE_ISS instead of EX1
6157 // Eg. ADD x0, x1, x2, LSL #2
6158 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6159 %{
6160 single_instruction;
6161 dst : EX2(write);
6162 src1 : EX1(read);
6163 src2 : ISS(read);
6164 INS01 : ISS;
6165 ALU : EX2;
6166 %}
6167
6168 // Integer ALU reg operation with constant shift
6169 // Eg. LSL x0, x1, #shift
6170 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6171 %{
6172 single_instruction;
6173 dst : EX2(write);
6174 src1 : ISS(read);
6175 INS01 : ISS;
6176 ALU : EX2;
6177 %}
6178
6179 // Integer ALU reg-reg operation with variable shift
6180 // Both operands must be available in LATE_ISS instead of EX1
6181 // Result is available in EX1 instead of EX2
6182 // Eg. LSLV x0, x1, x2
6183 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6184 %{
6185 single_instruction;
6186 dst : EX1(write);
6187 src1 : ISS(read);
6188 src2 : ISS(read);
6189 INS01 : ISS;
6190 ALU : EX1;
6191 %}
6192
6193 // Integer ALU reg-reg operation with extract
6194 // As for _vshift above, but result generated in EX2
6195 // Eg. EXTR x0, x1, x2, #N
6196 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6197 %{
6198 single_instruction;
6199 dst : EX2(write);
6200 src1 : ISS(read);
6201 src2 : ISS(read);
6202 INS1 : ISS; // Can only dual issue as Instruction 1
6203 ALU : EX1;
6204 %}
6205
6206 // Integer ALU reg operation
6207 // Eg. NEG x0, x1
6208 pipe_class ialu_reg(iRegI dst, iRegI src)
6209 %{
6210 single_instruction;
6211 dst : EX2(write);
6212 src : EX1(read);
6213 INS01 : ISS;
6214 ALU : EX2;
6215 %}
6216
6217 // Integer ALU reg mmediate operation
6218 // Eg. ADD x0, x1, #N
6219 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6220 %{
6221 single_instruction;
6222 dst : EX2(write);
6223 src1 : EX1(read);
6224 INS01 : ISS;
6225 ALU : EX2;
6226 %}
6227
6228 // Integer ALU immediate operation (no source operands)
6229 // Eg. MOV x0, #N
6230 pipe_class ialu_imm(iRegI dst)
6231 %{
6232 single_instruction;
6233 dst : EX1(write);
6234 INS01 : ISS;
6235 ALU : EX1;
6236 %}
6237
6238 //------- Compare operation -------------------------------
6239
6240 // Compare reg-reg
6241 // Eg. CMP x0, x1
6242 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6243 %{
6244 single_instruction;
6245 // fixed_latency(16);
6246 cr : EX2(write);
6247 op1 : EX1(read);
6248 op2 : EX1(read);
6249 INS01 : ISS;
6250 ALU : EX2;
6251 %}
6252
6253 // Compare reg-reg
6254 // Eg. CMP x0, #N
6255 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6256 %{
6257 single_instruction;
6258 // fixed_latency(16);
6259 cr : EX2(write);
6260 op1 : EX1(read);
6261 INS01 : ISS;
6262 ALU : EX2;
6263 %}
6264
6265 //------- Conditional instructions ------------------------
6266
6267 // Conditional no operands
6268 // Eg. CSINC x0, zr, zr, <cond>
6269 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6270 %{
6271 single_instruction;
6272 cr : EX1(read);
6273 dst : EX2(write);
6274 INS01 : ISS;
6275 ALU : EX2;
6276 %}
6277
6278 // Conditional 2 operand
6279 // EG. CSEL X0, X1, X2, <cond>
6280 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6281 %{
6282 single_instruction;
6283 cr : EX1(read);
6284 src1 : EX1(read);
6285 src2 : EX1(read);
6286 dst : EX2(write);
6287 INS01 : ISS;
6288 ALU : EX2;
6289 %}
6290
6291 // Conditional 2 operand
6292 // EG. CSEL X0, X1, X2, <cond>
6293 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6294 %{
6295 single_instruction;
6296 cr : EX1(read);
6297 src : EX1(read);
6298 dst : EX2(write);
6299 INS01 : ISS;
6300 ALU : EX2;
6301 %}
6302
6303 //------- Multiply pipeline operations --------------------
6304
6305 // Multiply reg-reg
6306 // Eg. MUL w0, w1, w2
6307 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6308 %{
6309 single_instruction;
6310 dst : WR(write);
6311 src1 : ISS(read);
6312 src2 : ISS(read);
6313 INS01 : ISS;
6314 MAC : WR;
6315 %}
6316
6317 // Multiply accumulate
6318 // Eg. MADD w0, w1, w2, w3
6319 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6320 %{
6321 single_instruction;
6322 dst : WR(write);
6323 src1 : ISS(read);
6324 src2 : ISS(read);
6325 src3 : ISS(read);
6326 INS01 : ISS;
6327 MAC : WR;
6328 %}
6329
6330 // Eg. MUL w0, w1, w2
6331 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6332 %{
6333 single_instruction;
6334 fixed_latency(3); // Maximum latency for 64 bit mul
6335 dst : WR(write);
6336 src1 : ISS(read);
6337 src2 : ISS(read);
6338 INS01 : ISS;
6339 MAC : WR;
6340 %}
6341
6342 // Multiply accumulate
6343 // Eg. MADD w0, w1, w2, w3
6344 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6345 %{
6346 single_instruction;
6347 fixed_latency(3); // Maximum latency for 64 bit mul
6348 dst : WR(write);
6349 src1 : ISS(read);
6350 src2 : ISS(read);
6351 src3 : ISS(read);
6352 INS01 : ISS;
6353 MAC : WR;
6354 %}
6355
6356 //------- Divide pipeline operations --------------------
6357
6358 // Eg. SDIV w0, w1, w2
6359 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6360 %{
6361 single_instruction;
6362 fixed_latency(8); // Maximum latency for 32 bit divide
6363 dst : WR(write);
6364 src1 : ISS(read);
6365 src2 : ISS(read);
6366 INS0 : ISS; // Can only dual issue as instruction 0
6367 DIV : WR;
6368 %}
6369
6370 // Eg. SDIV x0, x1, x2
6371 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6372 %{
6373 single_instruction;
6374 fixed_latency(16); // Maximum latency for 64 bit divide
6375 dst : WR(write);
6376 src1 : ISS(read);
6377 src2 : ISS(read);
6378 INS0 : ISS; // Can only dual issue as instruction 0
6379 DIV : WR;
6380 %}
6381
6382 //------- Load pipeline operations ------------------------
6383
6384 // Load - prefetch
6385 // Eg. PFRM <mem>
6386 pipe_class iload_prefetch(memory mem)
6387 %{
6388 single_instruction;
6389 mem : ISS(read);
6390 INS01 : ISS;
6391 LDST : WR;
6392 %}
6393
6394 // Load - reg, mem
6395 // Eg. LDR x0, <mem>
6396 pipe_class iload_reg_mem(iRegI dst, memory mem)
6397 %{
6398 single_instruction;
6399 dst : WR(write);
6400 mem : ISS(read);
6401 INS01 : ISS;
6402 LDST : WR;
6403 %}
6404
6405 // Load - reg, reg
6406 // Eg. LDR x0, [sp, x1]
6407 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6408 %{
6409 single_instruction;
6410 dst : WR(write);
6411 src : ISS(read);
6412 INS01 : ISS;
6413 LDST : WR;
6414 %}
6415
6416 //------- Store pipeline operations -----------------------
6417
6418 // Store - zr, mem
6419 // Eg. STR zr, <mem>
6420 pipe_class istore_mem(memory mem)
6421 %{
6422 single_instruction;
6423 mem : ISS(read);
6424 INS01 : ISS;
6425 LDST : WR;
6426 %}
6427
6428 // Store - reg, mem
6429 // Eg. STR x0, <mem>
6430 pipe_class istore_reg_mem(iRegI src, memory mem)
6431 %{
6432 single_instruction;
6433 mem : ISS(read);
6434 src : EX2(read);
6435 INS01 : ISS;
6436 LDST : WR;
6437 %}
6438
6439 // Store - reg, reg
6440 // Eg. STR x0, [sp, x1]
6441 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6442 %{
6443 single_instruction;
6444 dst : ISS(read);
6445 src : EX2(read);
6446 INS01 : ISS;
6447 LDST : WR;
6448 %}
6449
6450 //------- Store pipeline operations -----------------------
6451
6452 // Branch
6453 pipe_class pipe_branch()
6454 %{
6455 single_instruction;
6456 INS01 : ISS;
6457 BRANCH : EX1;
6458 %}
6459
6460 // Conditional branch
6461 pipe_class pipe_branch_cond(rFlagsReg cr)
6462 %{
6463 single_instruction;
6464 cr : EX1(read);
6465 INS01 : ISS;
6466 BRANCH : EX1;
6467 %}
6468
6469 // Compare & Branch
6470 // EG. CBZ/CBNZ
6471 pipe_class pipe_cmp_branch(iRegI op1)
6472 %{
6473 single_instruction;
6474 op1 : EX1(read);
6475 INS01 : ISS;
6476 BRANCH : EX1;
6477 %}
6478
6479 //------- Synchronisation operations ----------------------
6480
6481 // Any operation requiring serialization.
6482 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6483 pipe_class pipe_serial()
6484 %{
6485 single_instruction;
6486 force_serialization;
6487 fixed_latency(16);
6488 INS01 : ISS(2); // Cannot dual issue with any other instruction
6489 LDST : WR;
6490 %}
6491
6492 // Generic big/slow expanded idiom - also serialized
6493 pipe_class pipe_slow()
6494 %{
6495 instruction_count(10);
6496 multiple_bundles;
6497 force_serialization;
6498 fixed_latency(16);
6499 INS01 : ISS(2); // Cannot dual issue with any other instruction
6500 LDST : WR;
6501 %}
6502
6503 // Empty pipeline class
6504 pipe_class pipe_class_empty()
6505 %{
6506 single_instruction;
6507 fixed_latency(0);
6508 %}
6509
6510 // Default pipeline class.
6511 pipe_class pipe_class_default()
6512 %{
6513 single_instruction;
6514 fixed_latency(2);
6515 %}
6516
6517 // Pipeline class for compares.
6518 pipe_class pipe_class_compare()
6519 %{
6520 single_instruction;
6521 fixed_latency(16);
6522 %}
6523
6524 // Pipeline class for memory operations.
6525 pipe_class pipe_class_memory()
6526 %{
6527 single_instruction;
6528 fixed_latency(16);
6529 %}
6530
6531 // Pipeline class for call.
6532 pipe_class pipe_class_call()
6533 %{
6534 single_instruction;
6535 fixed_latency(100);
6536 %}
6537
6538 // Define the class for the Nop node.
6539 define %{
6540 MachNop = pipe_class_empty;
6541 %}
6542
6543 %}
6544 //----------INSTRUCTIONS-------------------------------------------------------
6545 //
6546 // match -- States which machine-independent subtree may be replaced
6547 // by this instruction.
6548 // ins_cost -- The estimated cost of this instruction is used by instruction
6549 // selection to identify a minimum cost tree of machine
6550 // instructions that matches a tree of machine-independent
6551 // instructions.
6552 // format -- A string providing the disassembly for this instruction.
6553 // The value of an instruction's operand may be inserted
6554 // by referring to it with a '$' prefix.
6555 // opcode -- Three instruction opcodes may be provided. These are referred
6556 // to within an encode class as $primary, $secondary, and $tertiary
6557 // rrspectively. The primary opcode is commonly used to
6558 // indicate the type of machine instruction, while secondary
6559 // and tertiary are often used for prefix options or addressing
6560 // modes.
6561 // ins_encode -- A list of encode classes with parameters. The encode class
6562 // name must have been defined in an 'enc_class' specification
6563 // in the encode section of the architecture description.
6564
6565 // ============================================================================
6566 // Memory (Load/Store) Instructions
6567
6568 // Load Instructions
6569
6570 // Load Byte (8 bit signed)
6571 instruct loadB(iRegINoSp dst, memory1 mem)
6572 %{
6573 match(Set dst (LoadB mem));
6574 predicate(!needs_acquiring_load(n));
6575
6576 ins_cost(4 * INSN_COST);
6577 format %{ "ldrsbw $dst, $mem\t# byte" %}
6578
6579 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6580
6581 ins_pipe(iload_reg_mem);
6582 %}
6583
6584 // Load Byte (8 bit signed) into long
6585 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6586 %{
6587 match(Set dst (ConvI2L (LoadB mem)));
6588 predicate(!needs_acquiring_load(n->in(1)));
6589
6590 ins_cost(4 * INSN_COST);
6591 format %{ "ldrsb $dst, $mem\t# byte" %}
6592
6593 ins_encode(aarch64_enc_ldrsb(dst, mem));
6594
6595 ins_pipe(iload_reg_mem);
6596 %}
6597
6598 // Load Byte (8 bit unsigned)
6599 instruct loadUB(iRegINoSp dst, memory1 mem)
6600 %{
6601 match(Set dst (LoadUB mem));
6602 predicate(!needs_acquiring_load(n));
6603
6604 ins_cost(4 * INSN_COST);
6605 format %{ "ldrbw $dst, $mem\t# byte" %}
6606
6607 ins_encode(aarch64_enc_ldrb(dst, mem));
6608
6609 ins_pipe(iload_reg_mem);
6610 %}
6611
6612 // Load Byte (8 bit unsigned) into long
6613 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6614 %{
6615 match(Set dst (ConvI2L (LoadUB mem)));
6616 predicate(!needs_acquiring_load(n->in(1)));
6617
6618 ins_cost(4 * INSN_COST);
6619 format %{ "ldrb $dst, $mem\t# byte" %}
6620
6621 ins_encode(aarch64_enc_ldrb(dst, mem));
6622
6623 ins_pipe(iload_reg_mem);
6624 %}
6625
6626 // Load Short (16 bit signed)
6627 instruct loadS(iRegINoSp dst, memory2 mem)
6628 %{
6629 match(Set dst (LoadS mem));
6630 predicate(!needs_acquiring_load(n));
6631
6632 ins_cost(4 * INSN_COST);
6633 format %{ "ldrshw $dst, $mem\t# short" %}
6634
6635 ins_encode(aarch64_enc_ldrshw(dst, mem));
6636
6637 ins_pipe(iload_reg_mem);
6638 %}
6639
6640 // Load Short (16 bit signed) into long
6641 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6642 %{
6643 match(Set dst (ConvI2L (LoadS mem)));
6644 predicate(!needs_acquiring_load(n->in(1)));
6645
6646 ins_cost(4 * INSN_COST);
6647 format %{ "ldrsh $dst, $mem\t# short" %}
6648
6649 ins_encode(aarch64_enc_ldrsh(dst, mem));
6650
6651 ins_pipe(iload_reg_mem);
6652 %}
6653
6654 // Load Char (16 bit unsigned)
6655 instruct loadUS(iRegINoSp dst, memory2 mem)
6656 %{
6657 match(Set dst (LoadUS mem));
6658 predicate(!needs_acquiring_load(n));
6659
6660 ins_cost(4 * INSN_COST);
6661 format %{ "ldrh $dst, $mem\t# short" %}
6662
6663 ins_encode(aarch64_enc_ldrh(dst, mem));
6664
6665 ins_pipe(iload_reg_mem);
6666 %}
6667
6668 // Load Short/Char (16 bit unsigned) into long
6669 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6670 %{
6671 match(Set dst (ConvI2L (LoadUS mem)));
6672 predicate(!needs_acquiring_load(n->in(1)));
6673
6674 ins_cost(4 * INSN_COST);
6675 format %{ "ldrh $dst, $mem\t# short" %}
6676
6677 ins_encode(aarch64_enc_ldrh(dst, mem));
6678
6679 ins_pipe(iload_reg_mem);
6680 %}
6681
6682 // Load Integer (32 bit signed)
6683 instruct loadI(iRegINoSp dst, memory4 mem)
6684 %{
6685 match(Set dst (LoadI mem));
6686 predicate(!needs_acquiring_load(n));
6687
6688 ins_cost(4 * INSN_COST);
6689 format %{ "ldrw $dst, $mem\t# int" %}
6690
6691 ins_encode(aarch64_enc_ldrw(dst, mem));
6692
6693 ins_pipe(iload_reg_mem);
6694 %}
6695
6696 // Load Integer (32 bit signed) into long
6697 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6698 %{
6699 match(Set dst (ConvI2L (LoadI mem)));
6700 predicate(!needs_acquiring_load(n->in(1)));
6701
6702 ins_cost(4 * INSN_COST);
6703 format %{ "ldrsw $dst, $mem\t# int" %}
6704
6705 ins_encode(aarch64_enc_ldrsw(dst, mem));
6706
6707 ins_pipe(iload_reg_mem);
6708 %}
6709
6710 // Load Integer (32 bit unsigned) into long
6711 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6712 %{
6713 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6714 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6715
6716 ins_cost(4 * INSN_COST);
6717 format %{ "ldrw $dst, $mem\t# int" %}
6718
6719 ins_encode(aarch64_enc_ldrw(dst, mem));
6720
6721 ins_pipe(iload_reg_mem);
6722 %}
6723
6724 // Load Long (64 bit signed)
6725 instruct loadL(iRegLNoSp dst, memory8 mem)
6726 %{
6727 match(Set dst (LoadL mem));
6728 predicate(!needs_acquiring_load(n));
6729
6730 ins_cost(4 * INSN_COST);
6731 format %{ "ldr $dst, $mem\t# int" %}
6732
6733 ins_encode(aarch64_enc_ldr(dst, mem));
6734
6735 ins_pipe(iload_reg_mem);
6736 %}
6737
6738 // Load Range
6739 instruct loadRange(iRegINoSp dst, memory4 mem)
6740 %{
6741 match(Set dst (LoadRange mem));
6742
6743 ins_cost(4 * INSN_COST);
6744 format %{ "ldrw $dst, $mem\t# range" %}
6745
6746 ins_encode(aarch64_enc_ldrw(dst, mem));
6747
6748 ins_pipe(iload_reg_mem);
6749 %}
6750
6751 // Load Pointer
6752 instruct loadP(iRegPNoSp dst, memory8 mem)
6753 %{
6754 match(Set dst (LoadP mem));
6755 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6756
6757 ins_cost(4 * INSN_COST);
6758 format %{ "ldr $dst, $mem\t# ptr" %}
6759
6760 ins_encode(aarch64_enc_ldr(dst, mem));
6761
6762 ins_pipe(iload_reg_mem);
6763 %}
6764
6765 // Load Compressed Pointer
6766 instruct loadN(iRegNNoSp dst, memory4 mem)
6767 %{
6768 match(Set dst (LoadN mem));
6769 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6770
6771 ins_cost(4 * INSN_COST);
6772 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6773
6774 ins_encode(aarch64_enc_ldrw(dst, mem));
6775
6776 ins_pipe(iload_reg_mem);
6777 %}
6778
6779 // Load Klass Pointer
6780 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6781 %{
6782 match(Set dst (LoadKlass mem));
6783 predicate(!needs_acquiring_load(n));
6784
6785 ins_cost(4 * INSN_COST);
6786 format %{ "ldr $dst, $mem\t# class" %}
6787
6788 ins_encode(aarch64_enc_ldr(dst, mem));
6789
6790 ins_pipe(iload_reg_mem);
6791 %}
6792
6793 // Load Narrow Klass Pointer
6794 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6795 %{
6796 match(Set dst (LoadNKlass mem));
6797 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6798
6799 ins_cost(4 * INSN_COST);
6800 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6801
6802 ins_encode(aarch64_enc_ldrw(dst, mem));
6803
6804 ins_pipe(iload_reg_mem);
6805 %}
6806
6807 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6808 %{
6809 match(Set dst (LoadNKlass mem));
6810 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6811
6812 ins_cost(4 * INSN_COST);
6813 format %{
6814 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6815 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6816 %}
6817 ins_encode %{
6818 // inlined aarch64_enc_ldrw
6819 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6820 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6821 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6822 %}
6823 ins_pipe(iload_reg_mem);
6824 %}
6825
6826 // Load Float
6827 instruct loadF(vRegF dst, memory4 mem)
6828 %{
6829 match(Set dst (LoadF mem));
6830 predicate(!needs_acquiring_load(n));
6831
6832 ins_cost(4 * INSN_COST);
6833 format %{ "ldrs $dst, $mem\t# float" %}
6834
6835 ins_encode( aarch64_enc_ldrs(dst, mem) );
6836
6837 ins_pipe(pipe_class_memory);
6838 %}
6839
6840 // Load Double
6841 instruct loadD(vRegD dst, memory8 mem)
6842 %{
6843 match(Set dst (LoadD mem));
6844 predicate(!needs_acquiring_load(n));
6845
6846 ins_cost(4 * INSN_COST);
6847 format %{ "ldrd $dst, $mem\t# double" %}
6848
6849 ins_encode( aarch64_enc_ldrd(dst, mem) );
6850
6851 ins_pipe(pipe_class_memory);
6852 %}
6853
6854
6855 // Load Int Constant
6856 instruct loadConI(iRegINoSp dst, immI src)
6857 %{
6858 match(Set dst src);
6859
6860 ins_cost(INSN_COST);
6861 format %{ "mov $dst, $src\t# int" %}
6862
6863 ins_encode( aarch64_enc_movw_imm(dst, src) );
6864
6865 ins_pipe(ialu_imm);
6866 %}
6867
6868 // Load Long Constant
6869 instruct loadConL(iRegLNoSp dst, immL src)
6870 %{
6871 match(Set dst src);
6872
6873 ins_cost(INSN_COST);
6874 format %{ "mov $dst, $src\t# long" %}
6875
6876 ins_encode( aarch64_enc_mov_imm(dst, src) );
6877
6878 ins_pipe(ialu_imm);
6879 %}
6880
6881 // Load Pointer Constant
6882
6883 instruct loadConP(iRegPNoSp dst, immP con)
6884 %{
6885 match(Set dst con);
6886
6887 ins_cost(INSN_COST * 4);
6888 format %{
6889 "mov $dst, $con\t# ptr"
6890 %}
6891
6892 ins_encode(aarch64_enc_mov_p(dst, con));
6893
6894 ins_pipe(ialu_imm);
6895 %}
6896
6897 // Load Null Pointer Constant
6898
6899 instruct loadConP0(iRegPNoSp dst, immP0 con)
6900 %{
6901 match(Set dst con);
6902
6903 ins_cost(INSN_COST);
6904 format %{ "mov $dst, $con\t# nullptr ptr" %}
6905
6906 ins_encode(aarch64_enc_mov_p0(dst, con));
6907
6908 ins_pipe(ialu_imm);
6909 %}
6910
6911 // Load Pointer Constant One
6912
6913 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6914 %{
6915 match(Set dst con);
6916
6917 ins_cost(INSN_COST);
6918 format %{ "mov $dst, $con\t# nullptr ptr" %}
6919
6920 ins_encode(aarch64_enc_mov_p1(dst, con));
6921
6922 ins_pipe(ialu_imm);
6923 %}
6924
6925 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6926 %{
6927 match(Set dst con);
6928
6929 ins_cost(INSN_COST);
6930 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6931
6932 ins_encode %{
6933 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6934 %}
6935
6936 ins_pipe(ialu_imm);
6937 %}
6938
6939 // Load Narrow Pointer Constant
6940
6941 instruct loadConN(iRegNNoSp dst, immN con)
6942 %{
6943 match(Set dst con);
6944
6945 ins_cost(INSN_COST * 4);
6946 format %{ "mov $dst, $con\t# compressed ptr" %}
6947
6948 ins_encode(aarch64_enc_mov_n(dst, con));
6949
6950 ins_pipe(ialu_imm);
6951 %}
6952
6953 // Load Narrow Null Pointer Constant
6954
6955 instruct loadConN0(iRegNNoSp dst, immN0 con)
6956 %{
6957 match(Set dst con);
6958
6959 ins_cost(INSN_COST);
6960 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6961
6962 ins_encode(aarch64_enc_mov_n0(dst, con));
6963
6964 ins_pipe(ialu_imm);
6965 %}
6966
6967 // Load Narrow Klass Constant
6968
6969 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6970 %{
6971 match(Set dst con);
6972
6973 ins_cost(INSN_COST);
6974 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6975
6976 ins_encode(aarch64_enc_mov_nk(dst, con));
6977
6978 ins_pipe(ialu_imm);
6979 %}
6980
6981 // Load Packed Float Constant
6982
6983 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6984 match(Set dst con);
6985 ins_cost(INSN_COST * 4);
6986 format %{ "fmovs $dst, $con"%}
6987 ins_encode %{
6988 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6989 %}
6990
6991 ins_pipe(fp_imm_s);
6992 %}
6993
6994 // Load Float Constant
6995
6996 instruct loadConF(vRegF dst, immF con) %{
6997 match(Set dst con);
6998
6999 ins_cost(INSN_COST * 4);
7000
7001 format %{
7002 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7003 %}
7004
7005 ins_encode %{
7006 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7007 %}
7008
7009 ins_pipe(fp_load_constant_s);
7010 %}
7011
7012 // Load Packed Double Constant
7013
7014 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7015 match(Set dst con);
7016 ins_cost(INSN_COST);
7017 format %{ "fmovd $dst, $con"%}
7018 ins_encode %{
7019 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7020 %}
7021
7022 ins_pipe(fp_imm_d);
7023 %}
7024
7025 // Load Double Constant
7026
7027 instruct loadConD(vRegD dst, immD con) %{
7028 match(Set dst con);
7029
7030 ins_cost(INSN_COST * 5);
7031 format %{
7032 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7033 %}
7034
7035 ins_encode %{
7036 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7037 %}
7038
7039 ins_pipe(fp_load_constant_d);
7040 %}
7041
7042 // Load Half Float Constant
7043 instruct loadConH(vRegF dst, immH con) %{
7044 match(Set dst con);
7045 format %{ "mov rscratch1, $con\n\t"
7046 "fmov $dst, rscratch1"
7047 %}
7048 ins_encode %{
7049 __ movw(rscratch1, (uint32_t)$con$$constant);
7050 __ fmovs($dst$$FloatRegister, rscratch1);
7051 %}
7052 ins_pipe(pipe_class_default);
7053 %}
7054
7055 // Store Instructions
7056
7057 // Store Byte
7058 instruct storeB(iRegIorL2I src, memory1 mem)
7059 %{
7060 match(Set mem (StoreB mem src));
7061 predicate(!needs_releasing_store(n));
7062
7063 ins_cost(INSN_COST);
7064 format %{ "strb $src, $mem\t# byte" %}
7065
7066 ins_encode(aarch64_enc_strb(src, mem));
7067
7068 ins_pipe(istore_reg_mem);
7069 %}
7070
7071
7072 instruct storeimmB0(immI0 zero, memory1 mem)
7073 %{
7074 match(Set mem (StoreB mem zero));
7075 predicate(!needs_releasing_store(n));
7076
7077 ins_cost(INSN_COST);
7078 format %{ "strb rscractch2, $mem\t# byte" %}
7079
7080 ins_encode(aarch64_enc_strb0(mem));
7081
7082 ins_pipe(istore_mem);
7083 %}
7084
7085 // Store Char/Short
7086 instruct storeC(iRegIorL2I src, memory2 mem)
7087 %{
7088 match(Set mem (StoreC mem src));
7089 predicate(!needs_releasing_store(n));
7090
7091 ins_cost(INSN_COST);
7092 format %{ "strh $src, $mem\t# short" %}
7093
7094 ins_encode(aarch64_enc_strh(src, mem));
7095
7096 ins_pipe(istore_reg_mem);
7097 %}
7098
7099 instruct storeimmC0(immI0 zero, memory2 mem)
7100 %{
7101 match(Set mem (StoreC mem zero));
7102 predicate(!needs_releasing_store(n));
7103
7104 ins_cost(INSN_COST);
7105 format %{ "strh zr, $mem\t# short" %}
7106
7107 ins_encode(aarch64_enc_strh0(mem));
7108
7109 ins_pipe(istore_mem);
7110 %}
7111
7112 // Store Integer
7113
7114 instruct storeI(iRegIorL2I src, memory4 mem)
7115 %{
7116 match(Set mem(StoreI mem src));
7117 predicate(!needs_releasing_store(n));
7118
7119 ins_cost(INSN_COST);
7120 format %{ "strw $src, $mem\t# int" %}
7121
7122 ins_encode(aarch64_enc_strw(src, mem));
7123
7124 ins_pipe(istore_reg_mem);
7125 %}
7126
7127 instruct storeimmI0(immI0 zero, memory4 mem)
7128 %{
7129 match(Set mem(StoreI mem zero));
7130 predicate(!needs_releasing_store(n));
7131
7132 ins_cost(INSN_COST);
7133 format %{ "strw zr, $mem\t# int" %}
7134
7135 ins_encode(aarch64_enc_strw0(mem));
7136
7137 ins_pipe(istore_mem);
7138 %}
7139
7140 // Store Long (64 bit signed)
7141 instruct storeL(iRegL src, memory8 mem)
7142 %{
7143 match(Set mem (StoreL mem src));
7144 predicate(!needs_releasing_store(n));
7145
7146 ins_cost(INSN_COST);
7147 format %{ "str $src, $mem\t# int" %}
7148
7149 ins_encode(aarch64_enc_str(src, mem));
7150
7151 ins_pipe(istore_reg_mem);
7152 %}
7153
7154 // Store Long (64 bit signed)
7155 instruct storeimmL0(immL0 zero, memory8 mem)
7156 %{
7157 match(Set mem (StoreL mem zero));
7158 predicate(!needs_releasing_store(n));
7159
7160 ins_cost(INSN_COST);
7161 format %{ "str zr, $mem\t# int" %}
7162
7163 ins_encode(aarch64_enc_str0(mem));
7164
7165 ins_pipe(istore_mem);
7166 %}
7167
7168 // Store Pointer
7169 instruct storeP(iRegP src, memory8 mem)
7170 %{
7171 match(Set mem (StoreP mem src));
7172 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7173
7174 ins_cost(INSN_COST);
7175 format %{ "str $src, $mem\t# ptr" %}
7176
7177 ins_encode(aarch64_enc_str(src, mem));
7178
7179 ins_pipe(istore_reg_mem);
7180 %}
7181
7182 // Store Pointer
7183 instruct storeimmP0(immP0 zero, memory8 mem)
7184 %{
7185 match(Set mem (StoreP mem zero));
7186 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7187
7188 ins_cost(INSN_COST);
7189 format %{ "str zr, $mem\t# ptr" %}
7190
7191 ins_encode(aarch64_enc_str0(mem));
7192
7193 ins_pipe(istore_mem);
7194 %}
7195
7196 // Store Compressed Pointer
7197 instruct storeN(iRegN src, memory4 mem)
7198 %{
7199 match(Set mem (StoreN mem src));
7200 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7201
7202 ins_cost(INSN_COST);
7203 format %{ "strw $src, $mem\t# compressed ptr" %}
7204
7205 ins_encode(aarch64_enc_strw(src, mem));
7206
7207 ins_pipe(istore_reg_mem);
7208 %}
7209
7210 instruct storeImmN0(immN0 zero, memory4 mem)
7211 %{
7212 match(Set mem (StoreN mem zero));
7213 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7214
7215 ins_cost(INSN_COST);
7216 format %{ "strw zr, $mem\t# compressed ptr" %}
7217
7218 ins_encode(aarch64_enc_strw0(mem));
7219
7220 ins_pipe(istore_mem);
7221 %}
7222
7223 // Store Float
7224 instruct storeF(vRegF src, memory4 mem)
7225 %{
7226 match(Set mem (StoreF mem src));
7227 predicate(!needs_releasing_store(n));
7228
7229 ins_cost(INSN_COST);
7230 format %{ "strs $src, $mem\t# float" %}
7231
7232 ins_encode( aarch64_enc_strs(src, mem) );
7233
7234 ins_pipe(pipe_class_memory);
7235 %}
7236
7237 // TODO
7238 // implement storeImmF0 and storeFImmPacked
7239
7240 // Store Double
7241 instruct storeD(vRegD src, memory8 mem)
7242 %{
7243 match(Set mem (StoreD mem src));
7244 predicate(!needs_releasing_store(n));
7245
7246 ins_cost(INSN_COST);
7247 format %{ "strd $src, $mem\t# double" %}
7248
7249 ins_encode( aarch64_enc_strd(src, mem) );
7250
7251 ins_pipe(pipe_class_memory);
7252 %}
7253
7254 // Store Compressed Klass Pointer
7255 instruct storeNKlass(iRegN src, memory4 mem)
7256 %{
7257 predicate(!needs_releasing_store(n));
7258 match(Set mem (StoreNKlass mem src));
7259
7260 ins_cost(INSN_COST);
7261 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7262
7263 ins_encode(aarch64_enc_strw(src, mem));
7264
7265 ins_pipe(istore_reg_mem);
7266 %}
7267
7268 // TODO
7269 // implement storeImmD0 and storeDImmPacked
7270
7271 // prefetch instructions
7272 // Must be safe to execute with invalid address (cannot fault).
7273
7274 instruct prefetchalloc( memory8 mem ) %{
7275 match(PrefetchAllocation mem);
7276
7277 ins_cost(INSN_COST);
7278 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7279
7280 ins_encode( aarch64_enc_prefetchw(mem) );
7281
7282 ins_pipe(iload_prefetch);
7283 %}
7284
7285 // ---------------- volatile loads and stores ----------------
7286
7287 // Load Byte (8 bit signed)
7288 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7289 %{
7290 match(Set dst (LoadB mem));
7291
7292 ins_cost(VOLATILE_REF_COST);
7293 format %{ "ldarsb $dst, $mem\t# byte" %}
7294
7295 ins_encode(aarch64_enc_ldarsb(dst, mem));
7296
7297 ins_pipe(pipe_serial);
7298 %}
7299
7300 // Load Byte (8 bit signed) into long
7301 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7302 %{
7303 match(Set dst (ConvI2L (LoadB mem)));
7304
7305 ins_cost(VOLATILE_REF_COST);
7306 format %{ "ldarsb $dst, $mem\t# byte" %}
7307
7308 ins_encode(aarch64_enc_ldarsb(dst, mem));
7309
7310 ins_pipe(pipe_serial);
7311 %}
7312
7313 // Load Byte (8 bit unsigned)
7314 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7315 %{
7316 match(Set dst (LoadUB mem));
7317
7318 ins_cost(VOLATILE_REF_COST);
7319 format %{ "ldarb $dst, $mem\t# byte" %}
7320
7321 ins_encode(aarch64_enc_ldarb(dst, mem));
7322
7323 ins_pipe(pipe_serial);
7324 %}
7325
7326 // Load Byte (8 bit unsigned) into long
7327 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7328 %{
7329 match(Set dst (ConvI2L (LoadUB mem)));
7330
7331 ins_cost(VOLATILE_REF_COST);
7332 format %{ "ldarb $dst, $mem\t# byte" %}
7333
7334 ins_encode(aarch64_enc_ldarb(dst, mem));
7335
7336 ins_pipe(pipe_serial);
7337 %}
7338
7339 // Load Short (16 bit signed)
7340 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7341 %{
7342 match(Set dst (LoadS mem));
7343
7344 ins_cost(VOLATILE_REF_COST);
7345 format %{ "ldarshw $dst, $mem\t# short" %}
7346
7347 ins_encode(aarch64_enc_ldarshw(dst, mem));
7348
7349 ins_pipe(pipe_serial);
7350 %}
7351
7352 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7353 %{
7354 match(Set dst (LoadUS mem));
7355
7356 ins_cost(VOLATILE_REF_COST);
7357 format %{ "ldarhw $dst, $mem\t# short" %}
7358
7359 ins_encode(aarch64_enc_ldarhw(dst, mem));
7360
7361 ins_pipe(pipe_serial);
7362 %}
7363
7364 // Load Short/Char (16 bit unsigned) into long
7365 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7366 %{
7367 match(Set dst (ConvI2L (LoadUS mem)));
7368
7369 ins_cost(VOLATILE_REF_COST);
7370 format %{ "ldarh $dst, $mem\t# short" %}
7371
7372 ins_encode(aarch64_enc_ldarh(dst, mem));
7373
7374 ins_pipe(pipe_serial);
7375 %}
7376
7377 // Load Short/Char (16 bit signed) into long
7378 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7379 %{
7380 match(Set dst (ConvI2L (LoadS mem)));
7381
7382 ins_cost(VOLATILE_REF_COST);
7383 format %{ "ldarh $dst, $mem\t# short" %}
7384
7385 ins_encode(aarch64_enc_ldarsh(dst, mem));
7386
7387 ins_pipe(pipe_serial);
7388 %}
7389
7390 // Load Integer (32 bit signed)
7391 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7392 %{
7393 match(Set dst (LoadI mem));
7394
7395 ins_cost(VOLATILE_REF_COST);
7396 format %{ "ldarw $dst, $mem\t# int" %}
7397
7398 ins_encode(aarch64_enc_ldarw(dst, mem));
7399
7400 ins_pipe(pipe_serial);
7401 %}
7402
7403 // Load Integer (32 bit unsigned) into long
7404 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7405 %{
7406 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7407
7408 ins_cost(VOLATILE_REF_COST);
7409 format %{ "ldarw $dst, $mem\t# int" %}
7410
7411 ins_encode(aarch64_enc_ldarw(dst, mem));
7412
7413 ins_pipe(pipe_serial);
7414 %}
7415
7416 // Load Long (64 bit signed)
7417 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7418 %{
7419 match(Set dst (LoadL mem));
7420
7421 ins_cost(VOLATILE_REF_COST);
7422 format %{ "ldar $dst, $mem\t# int" %}
7423
7424 ins_encode(aarch64_enc_ldar(dst, mem));
7425
7426 ins_pipe(pipe_serial);
7427 %}
7428
7429 // Load Pointer
7430 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7431 %{
7432 match(Set dst (LoadP mem));
7433 predicate(n->as_Load()->barrier_data() == 0);
7434
7435 ins_cost(VOLATILE_REF_COST);
7436 format %{ "ldar $dst, $mem\t# ptr" %}
7437
7438 ins_encode(aarch64_enc_ldar(dst, mem));
7439
7440 ins_pipe(pipe_serial);
7441 %}
7442
7443 // Load Compressed Pointer
7444 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7445 %{
7446 match(Set dst (LoadN mem));
7447 predicate(n->as_Load()->barrier_data() == 0);
7448
7449 ins_cost(VOLATILE_REF_COST);
7450 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7451
7452 ins_encode(aarch64_enc_ldarw(dst, mem));
7453
7454 ins_pipe(pipe_serial);
7455 %}
7456
7457 // Load Float
7458 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7459 %{
7460 match(Set dst (LoadF mem));
7461
7462 ins_cost(VOLATILE_REF_COST);
7463 format %{ "ldars $dst, $mem\t# float" %}
7464
7465 ins_encode( aarch64_enc_fldars(dst, mem) );
7466
7467 ins_pipe(pipe_serial);
7468 %}
7469
7470 // Load Double
7471 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7472 %{
7473 match(Set dst (LoadD mem));
7474
7475 ins_cost(VOLATILE_REF_COST);
7476 format %{ "ldard $dst, $mem\t# double" %}
7477
7478 ins_encode( aarch64_enc_fldard(dst, mem) );
7479
7480 ins_pipe(pipe_serial);
7481 %}
7482
7483 // Store Byte
7484 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7485 %{
7486 match(Set mem (StoreB mem src));
7487
7488 ins_cost(VOLATILE_REF_COST);
7489 format %{ "stlrb $src, $mem\t# byte" %}
7490
7491 ins_encode(aarch64_enc_stlrb(src, mem));
7492
7493 ins_pipe(pipe_class_memory);
7494 %}
7495
7496 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7497 %{
7498 match(Set mem (StoreB mem zero));
7499
7500 ins_cost(VOLATILE_REF_COST);
7501 format %{ "stlrb zr, $mem\t# byte" %}
7502
7503 ins_encode(aarch64_enc_stlrb0(mem));
7504
7505 ins_pipe(pipe_class_memory);
7506 %}
7507
7508 // Store Char/Short
7509 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7510 %{
7511 match(Set mem (StoreC mem src));
7512
7513 ins_cost(VOLATILE_REF_COST);
7514 format %{ "stlrh $src, $mem\t# short" %}
7515
7516 ins_encode(aarch64_enc_stlrh(src, mem));
7517
7518 ins_pipe(pipe_class_memory);
7519 %}
7520
7521 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7522 %{
7523 match(Set mem (StoreC mem zero));
7524
7525 ins_cost(VOLATILE_REF_COST);
7526 format %{ "stlrh zr, $mem\t# short" %}
7527
7528 ins_encode(aarch64_enc_stlrh0(mem));
7529
7530 ins_pipe(pipe_class_memory);
7531 %}
7532
7533 // Store Integer
7534
7535 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7536 %{
7537 match(Set mem(StoreI mem src));
7538
7539 ins_cost(VOLATILE_REF_COST);
7540 format %{ "stlrw $src, $mem\t# int" %}
7541
7542 ins_encode(aarch64_enc_stlrw(src, mem));
7543
7544 ins_pipe(pipe_class_memory);
7545 %}
7546
7547 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7548 %{
7549 match(Set mem(StoreI mem zero));
7550
7551 ins_cost(VOLATILE_REF_COST);
7552 format %{ "stlrw zr, $mem\t# int" %}
7553
7554 ins_encode(aarch64_enc_stlrw0(mem));
7555
7556 ins_pipe(pipe_class_memory);
7557 %}
7558
7559 // Store Long (64 bit signed)
7560 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7561 %{
7562 match(Set mem (StoreL mem src));
7563
7564 ins_cost(VOLATILE_REF_COST);
7565 format %{ "stlr $src, $mem\t# int" %}
7566
7567 ins_encode(aarch64_enc_stlr(src, mem));
7568
7569 ins_pipe(pipe_class_memory);
7570 %}
7571
7572 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7573 %{
7574 match(Set mem (StoreL mem zero));
7575
7576 ins_cost(VOLATILE_REF_COST);
7577 format %{ "stlr zr, $mem\t# int" %}
7578
7579 ins_encode(aarch64_enc_stlr0(mem));
7580
7581 ins_pipe(pipe_class_memory);
7582 %}
7583
7584 // Store Pointer
7585 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7586 %{
7587 match(Set mem (StoreP mem src));
7588 predicate(n->as_Store()->barrier_data() == 0);
7589
7590 ins_cost(VOLATILE_REF_COST);
7591 format %{ "stlr $src, $mem\t# ptr" %}
7592
7593 ins_encode(aarch64_enc_stlr(src, mem));
7594
7595 ins_pipe(pipe_class_memory);
7596 %}
7597
7598 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7599 %{
7600 match(Set mem (StoreP mem zero));
7601 predicate(n->as_Store()->barrier_data() == 0);
7602
7603 ins_cost(VOLATILE_REF_COST);
7604 format %{ "stlr zr, $mem\t# ptr" %}
7605
7606 ins_encode(aarch64_enc_stlr0(mem));
7607
7608 ins_pipe(pipe_class_memory);
7609 %}
7610
7611 // Store Compressed Pointer
7612 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7613 %{
7614 match(Set mem (StoreN mem src));
7615 predicate(n->as_Store()->barrier_data() == 0);
7616
7617 ins_cost(VOLATILE_REF_COST);
7618 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7619
7620 ins_encode(aarch64_enc_stlrw(src, mem));
7621
7622 ins_pipe(pipe_class_memory);
7623 %}
7624
7625 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7626 %{
7627 match(Set mem (StoreN mem zero));
7628 predicate(n->as_Store()->barrier_data() == 0);
7629
7630 ins_cost(VOLATILE_REF_COST);
7631 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7632
7633 ins_encode(aarch64_enc_stlrw0(mem));
7634
7635 ins_pipe(pipe_class_memory);
7636 %}
7637
7638 // Store Float
7639 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7640 %{
7641 match(Set mem (StoreF mem src));
7642
7643 ins_cost(VOLATILE_REF_COST);
7644 format %{ "stlrs $src, $mem\t# float" %}
7645
7646 ins_encode( aarch64_enc_fstlrs(src, mem) );
7647
7648 ins_pipe(pipe_class_memory);
7649 %}
7650
7651 // TODO
7652 // implement storeImmF0 and storeFImmPacked
7653
7654 // Store Double
7655 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7656 %{
7657 match(Set mem (StoreD mem src));
7658
7659 ins_cost(VOLATILE_REF_COST);
7660 format %{ "stlrd $src, $mem\t# double" %}
7661
7662 ins_encode( aarch64_enc_fstlrd(src, mem) );
7663
7664 ins_pipe(pipe_class_memory);
7665 %}
7666
7667 // ---------------- end of volatile loads and stores ----------------
7668
7669 instruct cacheWB(indirect addr)
7670 %{
7671 predicate(VM_Version::supports_data_cache_line_flush());
7672 match(CacheWB addr);
7673
7674 ins_cost(100);
7675 format %{"cache wb $addr" %}
7676 ins_encode %{
7677 assert($addr->index_position() < 0, "should be");
7678 assert($addr$$disp == 0, "should be");
7679 __ cache_wb(Address($addr$$base$$Register, 0));
7680 %}
7681 ins_pipe(pipe_slow); // XXX
7682 %}
7683
7684 instruct cacheWBPreSync()
7685 %{
7686 predicate(VM_Version::supports_data_cache_line_flush());
7687 match(CacheWBPreSync);
7688
7689 ins_cost(100);
7690 format %{"cache wb presync" %}
7691 ins_encode %{
7692 __ cache_wbsync(true);
7693 %}
7694 ins_pipe(pipe_slow); // XXX
7695 %}
7696
7697 instruct cacheWBPostSync()
7698 %{
7699 predicate(VM_Version::supports_data_cache_line_flush());
7700 match(CacheWBPostSync);
7701
7702 ins_cost(100);
7703 format %{"cache wb postsync" %}
7704 ins_encode %{
7705 __ cache_wbsync(false);
7706 %}
7707 ins_pipe(pipe_slow); // XXX
7708 %}
7709
7710 // ============================================================================
7711 // BSWAP Instructions
7712
7713 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7714 match(Set dst (ReverseBytesI src));
7715
7716 ins_cost(INSN_COST);
7717 format %{ "revw $dst, $src" %}
7718
7719 ins_encode %{
7720 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7721 %}
7722
7723 ins_pipe(ialu_reg);
7724 %}
7725
7726 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7727 match(Set dst (ReverseBytesL src));
7728
7729 ins_cost(INSN_COST);
7730 format %{ "rev $dst, $src" %}
7731
7732 ins_encode %{
7733 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7734 %}
7735
7736 ins_pipe(ialu_reg);
7737 %}
7738
7739 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7740 match(Set dst (ReverseBytesUS src));
7741
7742 ins_cost(INSN_COST);
7743 format %{ "rev16w $dst, $src" %}
7744
7745 ins_encode %{
7746 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7747 %}
7748
7749 ins_pipe(ialu_reg);
7750 %}
7751
7752 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7753 match(Set dst (ReverseBytesS src));
7754
7755 ins_cost(INSN_COST);
7756 format %{ "rev16w $dst, $src\n\t"
7757 "sbfmw $dst, $dst, #0, #15" %}
7758
7759 ins_encode %{
7760 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7761 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7762 %}
7763
7764 ins_pipe(ialu_reg);
7765 %}
7766
7767 // ============================================================================
7768 // Zero Count Instructions
7769
7770 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7771 match(Set dst (CountLeadingZerosI src));
7772
7773 ins_cost(INSN_COST);
7774 format %{ "clzw $dst, $src" %}
7775 ins_encode %{
7776 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7777 %}
7778
7779 ins_pipe(ialu_reg);
7780 %}
7781
7782 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7783 match(Set dst (CountLeadingZerosL src));
7784
7785 ins_cost(INSN_COST);
7786 format %{ "clz $dst, $src" %}
7787 ins_encode %{
7788 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7789 %}
7790
7791 ins_pipe(ialu_reg);
7792 %}
7793
7794 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7795 match(Set dst (CountTrailingZerosI src));
7796
7797 ins_cost(INSN_COST * 2);
7798 format %{ "rbitw $dst, $src\n\t"
7799 "clzw $dst, $dst" %}
7800 ins_encode %{
7801 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7802 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7803 %}
7804
7805 ins_pipe(ialu_reg);
7806 %}
7807
7808 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7809 match(Set dst (CountTrailingZerosL src));
7810
7811 ins_cost(INSN_COST * 2);
7812 format %{ "rbit $dst, $src\n\t"
7813 "clz $dst, $dst" %}
7814 ins_encode %{
7815 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7816 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7817 %}
7818
7819 ins_pipe(ialu_reg);
7820 %}
7821
7822 //---------- Population Count Instructions -------------------------------------
7823 //
7824
7825 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7826 match(Set dst (PopCountI src));
7827 effect(TEMP tmp);
7828 ins_cost(INSN_COST * 13);
7829
7830 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7831 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7832 "addv $tmp, $tmp\t# vector (8B)\n\t"
7833 "mov $dst, $tmp\t# vector (1D)" %}
7834 ins_encode %{
7835 __ fmovs($tmp$$FloatRegister, $src$$Register);
7836 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7837 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7838 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7839 %}
7840
7841 ins_pipe(pipe_class_default);
7842 %}
7843
7844 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7845 match(Set dst (PopCountI (LoadI mem)));
7846 effect(TEMP tmp);
7847 ins_cost(INSN_COST * 13);
7848
7849 format %{ "ldrs $tmp, $mem\n\t"
7850 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7851 "addv $tmp, $tmp\t# vector (8B)\n\t"
7852 "mov $dst, $tmp\t# vector (1D)" %}
7853 ins_encode %{
7854 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7855 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7856 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7857 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7858 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7859 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7860 %}
7861
7862 ins_pipe(pipe_class_default);
7863 %}
7864
7865 // Note: Long.bitCount(long) returns an int.
7866 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7867 match(Set dst (PopCountL src));
7868 effect(TEMP tmp);
7869 ins_cost(INSN_COST * 13);
7870
7871 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7872 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7873 "addv $tmp, $tmp\t# vector (8B)\n\t"
7874 "mov $dst, $tmp\t# vector (1D)" %}
7875 ins_encode %{
7876 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7877 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7878 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7879 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7880 %}
7881
7882 ins_pipe(pipe_class_default);
7883 %}
7884
7885 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7886 match(Set dst (PopCountL (LoadL mem)));
7887 effect(TEMP tmp);
7888 ins_cost(INSN_COST * 13);
7889
7890 format %{ "ldrd $tmp, $mem\n\t"
7891 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7892 "addv $tmp, $tmp\t# vector (8B)\n\t"
7893 "mov $dst, $tmp\t# vector (1D)" %}
7894 ins_encode %{
7895 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7896 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7897 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7898 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7899 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7900 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7901 %}
7902
7903 ins_pipe(pipe_class_default);
7904 %}
7905
7906 // ============================================================================
7907 // VerifyVectorAlignment Instruction
7908
7909 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7910 match(Set addr (VerifyVectorAlignment addr mask));
7911 effect(KILL cr);
7912 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7913 ins_encode %{
7914 Label Lskip;
7915 // check if masked bits of addr are zero
7916 __ tst($addr$$Register, $mask$$constant);
7917 __ br(Assembler::EQ, Lskip);
7918 __ stop("verify_vector_alignment found a misaligned vector memory access");
7919 __ bind(Lskip);
7920 %}
7921 ins_pipe(pipe_slow);
7922 %}
7923
7924 // ============================================================================
7925 // MemBar Instruction
7926
7927 instruct load_fence() %{
7928 match(LoadFence);
7929 ins_cost(VOLATILE_REF_COST);
7930
7931 format %{ "load_fence" %}
7932
7933 ins_encode %{
7934 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7935 %}
7936 ins_pipe(pipe_serial);
7937 %}
7938
7939 instruct unnecessary_membar_acquire() %{
7940 predicate(unnecessary_acquire(n));
7941 match(MemBarAcquire);
7942 ins_cost(0);
7943
7944 format %{ "membar_acquire (elided)" %}
7945
7946 ins_encode %{
7947 __ block_comment("membar_acquire (elided)");
7948 %}
7949
7950 ins_pipe(pipe_class_empty);
7951 %}
7952
7953 instruct membar_acquire() %{
7954 match(MemBarAcquire);
7955 ins_cost(VOLATILE_REF_COST);
7956
7957 format %{ "membar_acquire\n\t"
7958 "dmb ishld" %}
7959
7960 ins_encode %{
7961 __ block_comment("membar_acquire");
7962 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7963 %}
7964
7965 ins_pipe(pipe_serial);
7966 %}
7967
7968
7969 instruct membar_acquire_lock() %{
7970 match(MemBarAcquireLock);
7971 ins_cost(VOLATILE_REF_COST);
7972
7973 format %{ "membar_acquire_lock (elided)" %}
7974
7975 ins_encode %{
7976 __ block_comment("membar_acquire_lock (elided)");
7977 %}
7978
7979 ins_pipe(pipe_serial);
7980 %}
7981
7982 instruct store_fence() %{
7983 match(StoreFence);
7984 ins_cost(VOLATILE_REF_COST);
7985
7986 format %{ "store_fence" %}
7987
7988 ins_encode %{
7989 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7990 %}
7991 ins_pipe(pipe_serial);
7992 %}
7993
7994 instruct unnecessary_membar_release() %{
7995 predicate(unnecessary_release(n));
7996 match(MemBarRelease);
7997 ins_cost(0);
7998
7999 format %{ "membar_release (elided)" %}
8000
8001 ins_encode %{
8002 __ block_comment("membar_release (elided)");
8003 %}
8004 ins_pipe(pipe_serial);
8005 %}
8006
8007 instruct membar_release() %{
8008 match(MemBarRelease);
8009 ins_cost(VOLATILE_REF_COST);
8010
8011 format %{ "membar_release\n\t"
8012 "dmb ishst\n\tdmb ishld" %}
8013
8014 ins_encode %{
8015 __ block_comment("membar_release");
8016 // These will be merged if AlwaysMergeDMB is enabled.
8017 __ membar(Assembler::StoreStore);
8018 __ membar(Assembler::LoadStore);
8019 %}
8020 ins_pipe(pipe_serial);
8021 %}
8022
8023 instruct membar_storestore() %{
8024 match(MemBarStoreStore);
8025 match(StoreStoreFence);
8026 ins_cost(VOLATILE_REF_COST);
8027
8028 format %{ "MEMBAR-store-store" %}
8029
8030 ins_encode %{
8031 __ membar(Assembler::StoreStore);
8032 %}
8033 ins_pipe(pipe_serial);
8034 %}
8035
8036 instruct membar_release_lock() %{
8037 match(MemBarReleaseLock);
8038 ins_cost(VOLATILE_REF_COST);
8039
8040 format %{ "membar_release_lock (elided)" %}
8041
8042 ins_encode %{
8043 __ block_comment("membar_release_lock (elided)");
8044 %}
8045
8046 ins_pipe(pipe_serial);
8047 %}
8048
8049 instruct membar_storeload() %{
8050 match(MemBarStoreLoad);
8051 ins_cost(VOLATILE_REF_COST*100);
8052
8053 format %{ "MEMBAR-store-load\n\t"
8054 "dmb ish" %}
8055
8056 ins_encode %{
8057 __ block_comment("membar_storeload");
8058 __ membar(Assembler::StoreLoad);
8059 %}
8060
8061 ins_pipe(pipe_serial);
8062 %}
8063
8064 instruct unnecessary_membar_volatile() %{
8065 predicate(unnecessary_volatile(n));
8066 match(MemBarVolatile);
8067 ins_cost(0);
8068
8069 format %{ "membar_volatile (elided)" %}
8070
8071 ins_encode %{
8072 __ block_comment("membar_volatile (elided)");
8073 %}
8074
8075 ins_pipe(pipe_serial);
8076 %}
8077
8078 instruct membar_volatile() %{
8079 match(MemBarVolatile);
8080 ins_cost(VOLATILE_REF_COST*100);
8081
8082 format %{ "membar_volatile\n\t"
8083 "dmb ish"%}
8084
8085 ins_encode %{
8086 __ block_comment("membar_volatile");
8087 __ membar(Assembler::StoreLoad);
8088 %}
8089
8090 ins_pipe(pipe_serial);
8091 %}
8092
8093 instruct membar_full() %{
8094 match(MemBarFull);
8095 ins_cost(VOLATILE_REF_COST*100);
8096
8097 format %{ "membar_full\n\t"
8098 "dmb ish" %}
8099 ins_encode %{
8100 __ block_comment("membar_full");
8101 __ membar(Assembler::AnyAny);
8102 %}
8103
8104 ins_pipe(pipe_serial);
8105 %}
8106
8107 // ============================================================================
8108 // Cast/Convert Instructions
8109
8110 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8111 match(Set dst (CastX2P src));
8112
8113 ins_cost(INSN_COST);
8114 format %{ "mov $dst, $src\t# long -> ptr" %}
8115
8116 ins_encode %{
8117 if ($dst$$reg != $src$$reg) {
8118 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8119 }
8120 %}
8121
8122 ins_pipe(ialu_reg);
8123 %}
8124
8125 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8126 match(Set dst (CastI2N src));
8127
8128 ins_cost(INSN_COST);
8129 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8130
8131 ins_encode %{
8132 if ($dst$$reg != $src$$reg) {
8133 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8134 }
8135 %}
8136
8137 ins_pipe(ialu_reg);
8138 %}
8139
8140 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8141 match(Set dst (CastP2X src));
8142
8143 ins_cost(INSN_COST);
8144 format %{ "mov $dst, $src\t# ptr -> long" %}
8145
8146 ins_encode %{
8147 if ($dst$$reg != $src$$reg) {
8148 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8149 }
8150 %}
8151
8152 ins_pipe(ialu_reg);
8153 %}
8154
8155 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8156 match(Set dst (CastP2X src));
8157
8158 ins_cost(INSN_COST);
8159 format %{ "mov $dst, $src\t# ptr -> long" %}
8160
8161 ins_encode %{
8162 if ($dst$$reg != $src$$reg) {
8163 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8164 }
8165 %}
8166
8167 ins_pipe(ialu_reg);
8168 %}
8169
8170 // Convert oop into int for vectors alignment masking
8171 instruct convP2I(iRegINoSp dst, iRegP src) %{
8172 match(Set dst (ConvL2I (CastP2X src)));
8173
8174 ins_cost(INSN_COST);
8175 format %{ "movw $dst, $src\t# ptr -> int" %}
8176 ins_encode %{
8177 __ movw($dst$$Register, $src$$Register);
8178 %}
8179
8180 ins_pipe(ialu_reg);
8181 %}
8182
8183 // Convert compressed oop into int for vectors alignment masking
8184 // in case of 32bit oops (heap < 4Gb).
8185 instruct convN2I(iRegINoSp dst, iRegN src)
8186 %{
8187 predicate(CompressedOops::shift() == 0);
8188 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8189
8190 ins_cost(INSN_COST);
8191 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8192 ins_encode %{
8193 __ movw($dst$$Register, $src$$Register);
8194 %}
8195
8196 ins_pipe(ialu_reg);
8197 %}
8198
8199
8200 // Convert oop pointer into compressed form
8201 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8202 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8203 match(Set dst (EncodeP src));
8204 effect(KILL cr);
8205 ins_cost(INSN_COST * 3);
8206 format %{ "encode_heap_oop $dst, $src" %}
8207 ins_encode %{
8208 Register s = $src$$Register;
8209 Register d = $dst$$Register;
8210 __ encode_heap_oop(d, s);
8211 %}
8212 ins_pipe(ialu_reg);
8213 %}
8214
8215 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8216 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8217 match(Set dst (EncodeP src));
8218 ins_cost(INSN_COST * 3);
8219 format %{ "encode_heap_oop_not_null $dst, $src" %}
8220 ins_encode %{
8221 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8222 %}
8223 ins_pipe(ialu_reg);
8224 %}
8225
8226 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8227 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8228 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8229 match(Set dst (DecodeN src));
8230 ins_cost(INSN_COST * 3);
8231 format %{ "decode_heap_oop $dst, $src" %}
8232 ins_encode %{
8233 Register s = $src$$Register;
8234 Register d = $dst$$Register;
8235 __ decode_heap_oop(d, s);
8236 %}
8237 ins_pipe(ialu_reg);
8238 %}
8239
8240 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8241 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8242 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8243 match(Set dst (DecodeN src));
8244 ins_cost(INSN_COST * 3);
8245 format %{ "decode_heap_oop_not_null $dst, $src" %}
8246 ins_encode %{
8247 Register s = $src$$Register;
8248 Register d = $dst$$Register;
8249 __ decode_heap_oop_not_null(d, s);
8250 %}
8251 ins_pipe(ialu_reg);
8252 %}
8253
8254 // n.b. AArch64 implementations of encode_klass_not_null and
8255 // decode_klass_not_null do not modify the flags register so, unlike
8256 // Intel, we don't kill CR as a side effect here
8257
8258 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8259 match(Set dst (EncodePKlass src));
8260
8261 ins_cost(INSN_COST * 3);
8262 format %{ "encode_klass_not_null $dst,$src" %}
8263
8264 ins_encode %{
8265 Register src_reg = as_Register($src$$reg);
8266 Register dst_reg = as_Register($dst$$reg);
8267 __ encode_klass_not_null(dst_reg, src_reg);
8268 %}
8269
8270 ins_pipe(ialu_reg);
8271 %}
8272
8273 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8274 match(Set dst (DecodeNKlass src));
8275
8276 ins_cost(INSN_COST * 3);
8277 format %{ "decode_klass_not_null $dst,$src" %}
8278
8279 ins_encode %{
8280 Register src_reg = as_Register($src$$reg);
8281 Register dst_reg = as_Register($dst$$reg);
8282 if (dst_reg != src_reg) {
8283 __ decode_klass_not_null(dst_reg, src_reg);
8284 } else {
8285 __ decode_klass_not_null(dst_reg);
8286 }
8287 %}
8288
8289 ins_pipe(ialu_reg);
8290 %}
8291
8292 instruct checkCastPP(iRegPNoSp dst)
8293 %{
8294 match(Set dst (CheckCastPP dst));
8295
8296 size(0);
8297 format %{ "# checkcastPP of $dst" %}
8298 ins_encode(/* empty encoding */);
8299 ins_pipe(pipe_class_empty);
8300 %}
8301
8302 instruct castPP(iRegPNoSp dst)
8303 %{
8304 match(Set dst (CastPP dst));
8305
8306 size(0);
8307 format %{ "# castPP of $dst" %}
8308 ins_encode(/* empty encoding */);
8309 ins_pipe(pipe_class_empty);
8310 %}
8311
8312 instruct castII(iRegI dst)
8313 %{
8314 predicate(VerifyConstraintCasts == 0);
8315 match(Set dst (CastII dst));
8316
8317 size(0);
8318 format %{ "# castII of $dst" %}
8319 ins_encode(/* empty encoding */);
8320 ins_cost(0);
8321 ins_pipe(pipe_class_empty);
8322 %}
8323
8324 instruct castII_checked(iRegI dst, rFlagsReg cr)
8325 %{
8326 predicate(VerifyConstraintCasts > 0);
8327 match(Set dst (CastII dst));
8328 effect(KILL cr);
8329
8330 format %{ "# castII_checked of $dst" %}
8331 ins_encode %{
8332 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8333 %}
8334 ins_pipe(pipe_slow);
8335 %}
8336
8337 instruct castLL(iRegL dst)
8338 %{
8339 predicate(VerifyConstraintCasts == 0);
8340 match(Set dst (CastLL dst));
8341
8342 size(0);
8343 format %{ "# castLL of $dst" %}
8344 ins_encode(/* empty encoding */);
8345 ins_cost(0);
8346 ins_pipe(pipe_class_empty);
8347 %}
8348
8349 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8350 %{
8351 predicate(VerifyConstraintCasts > 0);
8352 match(Set dst (CastLL dst));
8353 effect(KILL cr);
8354
8355 format %{ "# castLL_checked of $dst" %}
8356 ins_encode %{
8357 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8358 %}
8359 ins_pipe(pipe_slow);
8360 %}
8361
8362 instruct castHH(vRegF dst)
8363 %{
8364 match(Set dst (CastHH dst));
8365 size(0);
8366 format %{ "# castHH of $dst" %}
8367 ins_encode(/* empty encoding */);
8368 ins_cost(0);
8369 ins_pipe(pipe_class_empty);
8370 %}
8371
8372 instruct castFF(vRegF dst)
8373 %{
8374 match(Set dst (CastFF dst));
8375
8376 size(0);
8377 format %{ "# castFF of $dst" %}
8378 ins_encode(/* empty encoding */);
8379 ins_cost(0);
8380 ins_pipe(pipe_class_empty);
8381 %}
8382
8383 instruct castDD(vRegD dst)
8384 %{
8385 match(Set dst (CastDD dst));
8386
8387 size(0);
8388 format %{ "# castDD of $dst" %}
8389 ins_encode(/* empty encoding */);
8390 ins_cost(0);
8391 ins_pipe(pipe_class_empty);
8392 %}
8393
8394 instruct castVV(vReg dst)
8395 %{
8396 match(Set dst (CastVV dst));
8397
8398 size(0);
8399 format %{ "# castVV of $dst" %}
8400 ins_encode(/* empty encoding */);
8401 ins_cost(0);
8402 ins_pipe(pipe_class_empty);
8403 %}
8404
8405 instruct castVVMask(pRegGov dst)
8406 %{
8407 match(Set dst (CastVV dst));
8408
8409 size(0);
8410 format %{ "# castVV of $dst" %}
8411 ins_encode(/* empty encoding */);
8412 ins_cost(0);
8413 ins_pipe(pipe_class_empty);
8414 %}
8415
8416 // Manifest a CmpU result in an integer register.
8417 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8418 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8419 %{
8420 match(Set dst (CmpU3 src1 src2));
8421 effect(KILL flags);
8422
8423 ins_cost(INSN_COST * 3);
8424 format %{
8425 "cmpw $src1, $src2\n\t"
8426 "csetw $dst, ne\n\t"
8427 "cnegw $dst, lo\t# CmpU3(reg)"
8428 %}
8429 ins_encode %{
8430 __ cmpw($src1$$Register, $src2$$Register);
8431 __ csetw($dst$$Register, Assembler::NE);
8432 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8433 %}
8434
8435 ins_pipe(pipe_class_default);
8436 %}
8437
8438 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8439 %{
8440 match(Set dst (CmpU3 src1 src2));
8441 effect(KILL flags);
8442
8443 ins_cost(INSN_COST * 3);
8444 format %{
8445 "subsw zr, $src1, $src2\n\t"
8446 "csetw $dst, ne\n\t"
8447 "cnegw $dst, lo\t# CmpU3(imm)"
8448 %}
8449 ins_encode %{
8450 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8451 __ csetw($dst$$Register, Assembler::NE);
8452 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8453 %}
8454
8455 ins_pipe(pipe_class_default);
8456 %}
8457
8458 // Manifest a CmpUL result in an integer register.
8459 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8460 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8461 %{
8462 match(Set dst (CmpUL3 src1 src2));
8463 effect(KILL flags);
8464
8465 ins_cost(INSN_COST * 3);
8466 format %{
8467 "cmp $src1, $src2\n\t"
8468 "csetw $dst, ne\n\t"
8469 "cnegw $dst, lo\t# CmpUL3(reg)"
8470 %}
8471 ins_encode %{
8472 __ cmp($src1$$Register, $src2$$Register);
8473 __ csetw($dst$$Register, Assembler::NE);
8474 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8475 %}
8476
8477 ins_pipe(pipe_class_default);
8478 %}
8479
8480 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8481 %{
8482 match(Set dst (CmpUL3 src1 src2));
8483 effect(KILL flags);
8484
8485 ins_cost(INSN_COST * 3);
8486 format %{
8487 "subs zr, $src1, $src2\n\t"
8488 "csetw $dst, ne\n\t"
8489 "cnegw $dst, lo\t# CmpUL3(imm)"
8490 %}
8491 ins_encode %{
8492 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8493 __ csetw($dst$$Register, Assembler::NE);
8494 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8495 %}
8496
8497 ins_pipe(pipe_class_default);
8498 %}
8499
8500 // Manifest a CmpL result in an integer register.
8501 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8502 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8503 %{
8504 match(Set dst (CmpL3 src1 src2));
8505 effect(KILL flags);
8506
8507 ins_cost(INSN_COST * 3);
8508 format %{
8509 "cmp $src1, $src2\n\t"
8510 "csetw $dst, ne\n\t"
8511 "cnegw $dst, lt\t# CmpL3(reg)"
8512 %}
8513 ins_encode %{
8514 __ cmp($src1$$Register, $src2$$Register);
8515 __ csetw($dst$$Register, Assembler::NE);
8516 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8517 %}
8518
8519 ins_pipe(pipe_class_default);
8520 %}
8521
8522 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8523 %{
8524 match(Set dst (CmpL3 src1 src2));
8525 effect(KILL flags);
8526
8527 ins_cost(INSN_COST * 3);
8528 format %{
8529 "subs zr, $src1, $src2\n\t"
8530 "csetw $dst, ne\n\t"
8531 "cnegw $dst, lt\t# CmpL3(imm)"
8532 %}
8533 ins_encode %{
8534 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8535 __ csetw($dst$$Register, Assembler::NE);
8536 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8537 %}
8538
8539 ins_pipe(pipe_class_default);
8540 %}
8541
8542 // ============================================================================
8543 // Conditional Move Instructions
8544
8545 // n.b. we have identical rules for both a signed compare op (cmpOp)
8546 // and an unsigned compare op (cmpOpU). it would be nice if we could
8547 // define an op class which merged both inputs and use it to type the
8548 // argument to a single rule. unfortunatelyt his fails because the
8549 // opclass does not live up to the COND_INTER interface of its
8550 // component operands. When the generic code tries to negate the
8551 // operand it ends up running the generci Machoper::negate method
8552 // which throws a ShouldNotHappen. So, we have to provide two flavours
8553 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8554
8555 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8556 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8557
8558 ins_cost(INSN_COST * 2);
8559 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8560
8561 ins_encode %{
8562 __ cselw(as_Register($dst$$reg),
8563 as_Register($src2$$reg),
8564 as_Register($src1$$reg),
8565 (Assembler::Condition)$cmp$$cmpcode);
8566 %}
8567
8568 ins_pipe(icond_reg_reg);
8569 %}
8570
8571 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8572 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8573
8574 ins_cost(INSN_COST * 2);
8575 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8576
8577 ins_encode %{
8578 __ cselw(as_Register($dst$$reg),
8579 as_Register($src2$$reg),
8580 as_Register($src1$$reg),
8581 (Assembler::Condition)$cmp$$cmpcode);
8582 %}
8583
8584 ins_pipe(icond_reg_reg);
8585 %}
8586
8587 // special cases where one arg is zero
8588
8589 // n.b. this is selected in preference to the rule above because it
8590 // avoids loading constant 0 into a source register
8591
8592 // TODO
8593 // we ought only to be able to cull one of these variants as the ideal
8594 // transforms ought always to order the zero consistently (to left/right?)
8595
8596 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8597 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8598
8599 ins_cost(INSN_COST * 2);
8600 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8601
8602 ins_encode %{
8603 __ cselw(as_Register($dst$$reg),
8604 as_Register($src$$reg),
8605 zr,
8606 (Assembler::Condition)$cmp$$cmpcode);
8607 %}
8608
8609 ins_pipe(icond_reg);
8610 %}
8611
8612 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8613 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8614
8615 ins_cost(INSN_COST * 2);
8616 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8617
8618 ins_encode %{
8619 __ cselw(as_Register($dst$$reg),
8620 as_Register($src$$reg),
8621 zr,
8622 (Assembler::Condition)$cmp$$cmpcode);
8623 %}
8624
8625 ins_pipe(icond_reg);
8626 %}
8627
8628 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8629 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8630
8631 ins_cost(INSN_COST * 2);
8632 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8633
8634 ins_encode %{
8635 __ cselw(as_Register($dst$$reg),
8636 zr,
8637 as_Register($src$$reg),
8638 (Assembler::Condition)$cmp$$cmpcode);
8639 %}
8640
8641 ins_pipe(icond_reg);
8642 %}
8643
8644 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8645 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8646
8647 ins_cost(INSN_COST * 2);
8648 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8649
8650 ins_encode %{
8651 __ cselw(as_Register($dst$$reg),
8652 zr,
8653 as_Register($src$$reg),
8654 (Assembler::Condition)$cmp$$cmpcode);
8655 %}
8656
8657 ins_pipe(icond_reg);
8658 %}
8659
8660 // special case for creating a boolean 0 or 1
8661
8662 // n.b. this is selected in preference to the rule above because it
8663 // avoids loading constants 0 and 1 into a source register
8664
8665 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8666 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8667
8668 ins_cost(INSN_COST * 2);
8669 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8670
8671 ins_encode %{
8672 // equivalently
8673 // cset(as_Register($dst$$reg),
8674 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8675 __ csincw(as_Register($dst$$reg),
8676 zr,
8677 zr,
8678 (Assembler::Condition)$cmp$$cmpcode);
8679 %}
8680
8681 ins_pipe(icond_none);
8682 %}
8683
8684 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8685 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8686
8687 ins_cost(INSN_COST * 2);
8688 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8689
8690 ins_encode %{
8691 // equivalently
8692 // cset(as_Register($dst$$reg),
8693 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8694 __ csincw(as_Register($dst$$reg),
8695 zr,
8696 zr,
8697 (Assembler::Condition)$cmp$$cmpcode);
8698 %}
8699
8700 ins_pipe(icond_none);
8701 %}
8702
8703 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8704 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8705
8706 ins_cost(INSN_COST * 2);
8707 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8708
8709 ins_encode %{
8710 __ csel(as_Register($dst$$reg),
8711 as_Register($src2$$reg),
8712 as_Register($src1$$reg),
8713 (Assembler::Condition)$cmp$$cmpcode);
8714 %}
8715
8716 ins_pipe(icond_reg_reg);
8717 %}
8718
8719 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8720 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8721
8722 ins_cost(INSN_COST * 2);
8723 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8724
8725 ins_encode %{
8726 __ csel(as_Register($dst$$reg),
8727 as_Register($src2$$reg),
8728 as_Register($src1$$reg),
8729 (Assembler::Condition)$cmp$$cmpcode);
8730 %}
8731
8732 ins_pipe(icond_reg_reg);
8733 %}
8734
8735 // special cases where one arg is zero
8736
8737 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8738 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8739
8740 ins_cost(INSN_COST * 2);
8741 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8742
8743 ins_encode %{
8744 __ csel(as_Register($dst$$reg),
8745 zr,
8746 as_Register($src$$reg),
8747 (Assembler::Condition)$cmp$$cmpcode);
8748 %}
8749
8750 ins_pipe(icond_reg);
8751 %}
8752
8753 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8754 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8755
8756 ins_cost(INSN_COST * 2);
8757 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8758
8759 ins_encode %{
8760 __ csel(as_Register($dst$$reg),
8761 zr,
8762 as_Register($src$$reg),
8763 (Assembler::Condition)$cmp$$cmpcode);
8764 %}
8765
8766 ins_pipe(icond_reg);
8767 %}
8768
8769 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8770 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8771
8772 ins_cost(INSN_COST * 2);
8773 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8774
8775 ins_encode %{
8776 __ csel(as_Register($dst$$reg),
8777 as_Register($src$$reg),
8778 zr,
8779 (Assembler::Condition)$cmp$$cmpcode);
8780 %}
8781
8782 ins_pipe(icond_reg);
8783 %}
8784
8785 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8786 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8787
8788 ins_cost(INSN_COST * 2);
8789 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8790
8791 ins_encode %{
8792 __ csel(as_Register($dst$$reg),
8793 as_Register($src$$reg),
8794 zr,
8795 (Assembler::Condition)$cmp$$cmpcode);
8796 %}
8797
8798 ins_pipe(icond_reg);
8799 %}
8800
8801 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8802 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8803
8804 ins_cost(INSN_COST * 2);
8805 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8806
8807 ins_encode %{
8808 __ csel(as_Register($dst$$reg),
8809 as_Register($src2$$reg),
8810 as_Register($src1$$reg),
8811 (Assembler::Condition)$cmp$$cmpcode);
8812 %}
8813
8814 ins_pipe(icond_reg_reg);
8815 %}
8816
8817 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8818 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8819
8820 ins_cost(INSN_COST * 2);
8821 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8822
8823 ins_encode %{
8824 __ csel(as_Register($dst$$reg),
8825 as_Register($src2$$reg),
8826 as_Register($src1$$reg),
8827 (Assembler::Condition)$cmp$$cmpcode);
8828 %}
8829
8830 ins_pipe(icond_reg_reg);
8831 %}
8832
8833 // special cases where one arg is zero
8834
8835 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8836 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8837
8838 ins_cost(INSN_COST * 2);
8839 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8840
8841 ins_encode %{
8842 __ csel(as_Register($dst$$reg),
8843 zr,
8844 as_Register($src$$reg),
8845 (Assembler::Condition)$cmp$$cmpcode);
8846 %}
8847
8848 ins_pipe(icond_reg);
8849 %}
8850
8851 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8852 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8853
8854 ins_cost(INSN_COST * 2);
8855 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8856
8857 ins_encode %{
8858 __ csel(as_Register($dst$$reg),
8859 zr,
8860 as_Register($src$$reg),
8861 (Assembler::Condition)$cmp$$cmpcode);
8862 %}
8863
8864 ins_pipe(icond_reg);
8865 %}
8866
8867 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8868 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8869
8870 ins_cost(INSN_COST * 2);
8871 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8872
8873 ins_encode %{
8874 __ csel(as_Register($dst$$reg),
8875 as_Register($src$$reg),
8876 zr,
8877 (Assembler::Condition)$cmp$$cmpcode);
8878 %}
8879
8880 ins_pipe(icond_reg);
8881 %}
8882
8883 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8884 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8885
8886 ins_cost(INSN_COST * 2);
8887 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8888
8889 ins_encode %{
8890 __ csel(as_Register($dst$$reg),
8891 as_Register($src$$reg),
8892 zr,
8893 (Assembler::Condition)$cmp$$cmpcode);
8894 %}
8895
8896 ins_pipe(icond_reg);
8897 %}
8898
8899 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8900 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8901
8902 ins_cost(INSN_COST * 2);
8903 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8904
8905 ins_encode %{
8906 __ cselw(as_Register($dst$$reg),
8907 as_Register($src2$$reg),
8908 as_Register($src1$$reg),
8909 (Assembler::Condition)$cmp$$cmpcode);
8910 %}
8911
8912 ins_pipe(icond_reg_reg);
8913 %}
8914
8915 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8916 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8917
8918 ins_cost(INSN_COST * 2);
8919 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8920
8921 ins_encode %{
8922 __ cselw(as_Register($dst$$reg),
8923 as_Register($src2$$reg),
8924 as_Register($src1$$reg),
8925 (Assembler::Condition)$cmp$$cmpcode);
8926 %}
8927
8928 ins_pipe(icond_reg_reg);
8929 %}
8930
8931 // special cases where one arg is zero
8932
8933 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8934 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8935
8936 ins_cost(INSN_COST * 2);
8937 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8938
8939 ins_encode %{
8940 __ cselw(as_Register($dst$$reg),
8941 zr,
8942 as_Register($src$$reg),
8943 (Assembler::Condition)$cmp$$cmpcode);
8944 %}
8945
8946 ins_pipe(icond_reg);
8947 %}
8948
8949 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8950 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8951
8952 ins_cost(INSN_COST * 2);
8953 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8954
8955 ins_encode %{
8956 __ cselw(as_Register($dst$$reg),
8957 zr,
8958 as_Register($src$$reg),
8959 (Assembler::Condition)$cmp$$cmpcode);
8960 %}
8961
8962 ins_pipe(icond_reg);
8963 %}
8964
8965 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8966 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8967
8968 ins_cost(INSN_COST * 2);
8969 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8970
8971 ins_encode %{
8972 __ cselw(as_Register($dst$$reg),
8973 as_Register($src$$reg),
8974 zr,
8975 (Assembler::Condition)$cmp$$cmpcode);
8976 %}
8977
8978 ins_pipe(icond_reg);
8979 %}
8980
8981 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8982 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8983
8984 ins_cost(INSN_COST * 2);
8985 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8986
8987 ins_encode %{
8988 __ cselw(as_Register($dst$$reg),
8989 as_Register($src$$reg),
8990 zr,
8991 (Assembler::Condition)$cmp$$cmpcode);
8992 %}
8993
8994 ins_pipe(icond_reg);
8995 %}
8996
8997 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8998 %{
8999 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9000
9001 ins_cost(INSN_COST * 3);
9002
9003 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9004 ins_encode %{
9005 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9006 __ fcsels(as_FloatRegister($dst$$reg),
9007 as_FloatRegister($src2$$reg),
9008 as_FloatRegister($src1$$reg),
9009 cond);
9010 %}
9011
9012 ins_pipe(fp_cond_reg_reg_s);
9013 %}
9014
9015 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9016 %{
9017 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9018
9019 ins_cost(INSN_COST * 3);
9020
9021 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9022 ins_encode %{
9023 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9024 __ fcsels(as_FloatRegister($dst$$reg),
9025 as_FloatRegister($src2$$reg),
9026 as_FloatRegister($src1$$reg),
9027 cond);
9028 %}
9029
9030 ins_pipe(fp_cond_reg_reg_s);
9031 %}
9032
9033 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9034 %{
9035 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9036
9037 ins_cost(INSN_COST * 3);
9038
9039 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9040 ins_encode %{
9041 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9042 __ fcseld(as_FloatRegister($dst$$reg),
9043 as_FloatRegister($src2$$reg),
9044 as_FloatRegister($src1$$reg),
9045 cond);
9046 %}
9047
9048 ins_pipe(fp_cond_reg_reg_d);
9049 %}
9050
9051 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9052 %{
9053 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9054
9055 ins_cost(INSN_COST * 3);
9056
9057 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9058 ins_encode %{
9059 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9060 __ fcseld(as_FloatRegister($dst$$reg),
9061 as_FloatRegister($src2$$reg),
9062 as_FloatRegister($src1$$reg),
9063 cond);
9064 %}
9065
9066 ins_pipe(fp_cond_reg_reg_d);
9067 %}
9068
9069 // ============================================================================
9070 // Arithmetic Instructions
9071 //
9072
9073 // Integer Addition
9074
9075 // TODO
9076 // these currently employ operations which do not set CR and hence are
9077 // not flagged as killing CR but we would like to isolate the cases
9078 // where we want to set flags from those where we don't. need to work
9079 // out how to do that.
9080
9081 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9082 match(Set dst (AddI src1 src2));
9083
9084 ins_cost(INSN_COST);
9085 format %{ "addw $dst, $src1, $src2" %}
9086
9087 ins_encode %{
9088 __ addw(as_Register($dst$$reg),
9089 as_Register($src1$$reg),
9090 as_Register($src2$$reg));
9091 %}
9092
9093 ins_pipe(ialu_reg_reg);
9094 %}
9095
9096 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9097 match(Set dst (AddI src1 src2));
9098
9099 ins_cost(INSN_COST);
9100 format %{ "addw $dst, $src1, $src2" %}
9101
9102 // use opcode to indicate that this is an add not a sub
9103 opcode(0x0);
9104
9105 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9106
9107 ins_pipe(ialu_reg_imm);
9108 %}
9109
9110 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9111 match(Set dst (AddI (ConvL2I src1) src2));
9112
9113 ins_cost(INSN_COST);
9114 format %{ "addw $dst, $src1, $src2" %}
9115
9116 // use opcode to indicate that this is an add not a sub
9117 opcode(0x0);
9118
9119 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9120
9121 ins_pipe(ialu_reg_imm);
9122 %}
9123
9124 // Pointer Addition
9125 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9126 match(Set dst (AddP src1 src2));
9127
9128 ins_cost(INSN_COST);
9129 format %{ "add $dst, $src1, $src2\t# ptr" %}
9130
9131 ins_encode %{
9132 __ add(as_Register($dst$$reg),
9133 as_Register($src1$$reg),
9134 as_Register($src2$$reg));
9135 %}
9136
9137 ins_pipe(ialu_reg_reg);
9138 %}
9139
9140 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9141 match(Set dst (AddP src1 (ConvI2L src2)));
9142
9143 ins_cost(1.9 * INSN_COST);
9144 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9145
9146 ins_encode %{
9147 __ add(as_Register($dst$$reg),
9148 as_Register($src1$$reg),
9149 as_Register($src2$$reg), ext::sxtw);
9150 %}
9151
9152 ins_pipe(ialu_reg_reg);
9153 %}
9154
9155 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9156 match(Set dst (AddP src1 (LShiftL src2 scale)));
9157
9158 ins_cost(1.9 * INSN_COST);
9159 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9160
9161 ins_encode %{
9162 __ lea(as_Register($dst$$reg),
9163 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9164 Address::lsl($scale$$constant)));
9165 %}
9166
9167 ins_pipe(ialu_reg_reg_shift);
9168 %}
9169
9170 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9171 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9172
9173 ins_cost(1.9 * INSN_COST);
9174 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9175
9176 ins_encode %{
9177 __ lea(as_Register($dst$$reg),
9178 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9179 Address::sxtw($scale$$constant)));
9180 %}
9181
9182 ins_pipe(ialu_reg_reg_shift);
9183 %}
9184
9185 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9186 match(Set dst (LShiftL (ConvI2L src) scale));
9187
9188 ins_cost(INSN_COST);
9189 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9190
9191 ins_encode %{
9192 __ sbfiz(as_Register($dst$$reg),
9193 as_Register($src$$reg),
9194 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9195 %}
9196
9197 ins_pipe(ialu_reg_shift);
9198 %}
9199
9200 // Pointer Immediate Addition
9201 // n.b. this needs to be more expensive than using an indirect memory
9202 // operand
9203 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9204 match(Set dst (AddP src1 src2));
9205
9206 ins_cost(INSN_COST);
9207 format %{ "add $dst, $src1, $src2\t# ptr" %}
9208
9209 // use opcode to indicate that this is an add not a sub
9210 opcode(0x0);
9211
9212 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9213
9214 ins_pipe(ialu_reg_imm);
9215 %}
9216
9217 // Long Addition
9218 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9219
9220 match(Set dst (AddL src1 src2));
9221
9222 ins_cost(INSN_COST);
9223 format %{ "add $dst, $src1, $src2" %}
9224
9225 ins_encode %{
9226 __ add(as_Register($dst$$reg),
9227 as_Register($src1$$reg),
9228 as_Register($src2$$reg));
9229 %}
9230
9231 ins_pipe(ialu_reg_reg);
9232 %}
9233
9234 // No constant pool entries requiredLong Immediate Addition.
9235 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9236 match(Set dst (AddL src1 src2));
9237
9238 ins_cost(INSN_COST);
9239 format %{ "add $dst, $src1, $src2" %}
9240
9241 // use opcode to indicate that this is an add not a sub
9242 opcode(0x0);
9243
9244 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9245
9246 ins_pipe(ialu_reg_imm);
9247 %}
9248
9249 // Integer Subtraction
9250 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9251 match(Set dst (SubI src1 src2));
9252
9253 ins_cost(INSN_COST);
9254 format %{ "subw $dst, $src1, $src2" %}
9255
9256 ins_encode %{
9257 __ subw(as_Register($dst$$reg),
9258 as_Register($src1$$reg),
9259 as_Register($src2$$reg));
9260 %}
9261
9262 ins_pipe(ialu_reg_reg);
9263 %}
9264
9265 // Immediate Subtraction
9266 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9267 match(Set dst (SubI src1 src2));
9268
9269 ins_cost(INSN_COST);
9270 format %{ "subw $dst, $src1, $src2" %}
9271
9272 // use opcode to indicate that this is a sub not an add
9273 opcode(0x1);
9274
9275 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9276
9277 ins_pipe(ialu_reg_imm);
9278 %}
9279
9280 // Long Subtraction
9281 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9282
9283 match(Set dst (SubL src1 src2));
9284
9285 ins_cost(INSN_COST);
9286 format %{ "sub $dst, $src1, $src2" %}
9287
9288 ins_encode %{
9289 __ sub(as_Register($dst$$reg),
9290 as_Register($src1$$reg),
9291 as_Register($src2$$reg));
9292 %}
9293
9294 ins_pipe(ialu_reg_reg);
9295 %}
9296
9297 // No constant pool entries requiredLong Immediate Subtraction.
9298 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9299 match(Set dst (SubL src1 src2));
9300
9301 ins_cost(INSN_COST);
9302 format %{ "sub$dst, $src1, $src2" %}
9303
9304 // use opcode to indicate that this is a sub not an add
9305 opcode(0x1);
9306
9307 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9308
9309 ins_pipe(ialu_reg_imm);
9310 %}
9311
9312 // Integer Negation (special case for sub)
9313
9314 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9315 match(Set dst (SubI zero src));
9316
9317 ins_cost(INSN_COST);
9318 format %{ "negw $dst, $src\t# int" %}
9319
9320 ins_encode %{
9321 __ negw(as_Register($dst$$reg),
9322 as_Register($src$$reg));
9323 %}
9324
9325 ins_pipe(ialu_reg);
9326 %}
9327
9328 // Long Negation
9329
9330 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9331 match(Set dst (SubL zero src));
9332
9333 ins_cost(INSN_COST);
9334 format %{ "neg $dst, $src\t# long" %}
9335
9336 ins_encode %{
9337 __ neg(as_Register($dst$$reg),
9338 as_Register($src$$reg));
9339 %}
9340
9341 ins_pipe(ialu_reg);
9342 %}
9343
9344 // Integer Multiply
9345
9346 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9347 match(Set dst (MulI src1 src2));
9348
9349 ins_cost(INSN_COST * 3);
9350 format %{ "mulw $dst, $src1, $src2" %}
9351
9352 ins_encode %{
9353 __ mulw(as_Register($dst$$reg),
9354 as_Register($src1$$reg),
9355 as_Register($src2$$reg));
9356 %}
9357
9358 ins_pipe(imul_reg_reg);
9359 %}
9360
9361 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9362 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9363
9364 ins_cost(INSN_COST * 3);
9365 format %{ "smull $dst, $src1, $src2" %}
9366
9367 ins_encode %{
9368 __ smull(as_Register($dst$$reg),
9369 as_Register($src1$$reg),
9370 as_Register($src2$$reg));
9371 %}
9372
9373 ins_pipe(imul_reg_reg);
9374 %}
9375
9376 // Long Multiply
9377
9378 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9379 match(Set dst (MulL src1 src2));
9380
9381 ins_cost(INSN_COST * 5);
9382 format %{ "mul $dst, $src1, $src2" %}
9383
9384 ins_encode %{
9385 __ mul(as_Register($dst$$reg),
9386 as_Register($src1$$reg),
9387 as_Register($src2$$reg));
9388 %}
9389
9390 ins_pipe(lmul_reg_reg);
9391 %}
9392
9393 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9394 %{
9395 match(Set dst (MulHiL src1 src2));
9396
9397 ins_cost(INSN_COST * 7);
9398 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9399
9400 ins_encode %{
9401 __ smulh(as_Register($dst$$reg),
9402 as_Register($src1$$reg),
9403 as_Register($src2$$reg));
9404 %}
9405
9406 ins_pipe(lmul_reg_reg);
9407 %}
9408
9409 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9410 %{
9411 match(Set dst (UMulHiL src1 src2));
9412
9413 ins_cost(INSN_COST * 7);
9414 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9415
9416 ins_encode %{
9417 __ umulh(as_Register($dst$$reg),
9418 as_Register($src1$$reg),
9419 as_Register($src2$$reg));
9420 %}
9421
9422 ins_pipe(lmul_reg_reg);
9423 %}
9424
9425 // Combined Integer Multiply & Add/Sub
9426
9427 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9428 match(Set dst (AddI src3 (MulI src1 src2)));
9429
9430 ins_cost(INSN_COST * 3);
9431 format %{ "madd $dst, $src1, $src2, $src3" %}
9432
9433 ins_encode %{
9434 __ maddw(as_Register($dst$$reg),
9435 as_Register($src1$$reg),
9436 as_Register($src2$$reg),
9437 as_Register($src3$$reg));
9438 %}
9439
9440 ins_pipe(imac_reg_reg);
9441 %}
9442
9443 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9444 match(Set dst (SubI src3 (MulI src1 src2)));
9445
9446 ins_cost(INSN_COST * 3);
9447 format %{ "msub $dst, $src1, $src2, $src3" %}
9448
9449 ins_encode %{
9450 __ msubw(as_Register($dst$$reg),
9451 as_Register($src1$$reg),
9452 as_Register($src2$$reg),
9453 as_Register($src3$$reg));
9454 %}
9455
9456 ins_pipe(imac_reg_reg);
9457 %}
9458
9459 // Combined Integer Multiply & Neg
9460
9461 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9462 match(Set dst (MulI (SubI zero src1) src2));
9463
9464 ins_cost(INSN_COST * 3);
9465 format %{ "mneg $dst, $src1, $src2" %}
9466
9467 ins_encode %{
9468 __ mnegw(as_Register($dst$$reg),
9469 as_Register($src1$$reg),
9470 as_Register($src2$$reg));
9471 %}
9472
9473 ins_pipe(imac_reg_reg);
9474 %}
9475
9476 // Combined Long Multiply & Add/Sub
9477
9478 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9479 match(Set dst (AddL src3 (MulL src1 src2)));
9480
9481 ins_cost(INSN_COST * 5);
9482 format %{ "madd $dst, $src1, $src2, $src3" %}
9483
9484 ins_encode %{
9485 __ madd(as_Register($dst$$reg),
9486 as_Register($src1$$reg),
9487 as_Register($src2$$reg),
9488 as_Register($src3$$reg));
9489 %}
9490
9491 ins_pipe(lmac_reg_reg);
9492 %}
9493
9494 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9495 match(Set dst (SubL src3 (MulL src1 src2)));
9496
9497 ins_cost(INSN_COST * 5);
9498 format %{ "msub $dst, $src1, $src2, $src3" %}
9499
9500 ins_encode %{
9501 __ msub(as_Register($dst$$reg),
9502 as_Register($src1$$reg),
9503 as_Register($src2$$reg),
9504 as_Register($src3$$reg));
9505 %}
9506
9507 ins_pipe(lmac_reg_reg);
9508 %}
9509
9510 // Combined Long Multiply & Neg
9511
9512 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9513 match(Set dst (MulL (SubL zero src1) src2));
9514
9515 ins_cost(INSN_COST * 5);
9516 format %{ "mneg $dst, $src1, $src2" %}
9517
9518 ins_encode %{
9519 __ mneg(as_Register($dst$$reg),
9520 as_Register($src1$$reg),
9521 as_Register($src2$$reg));
9522 %}
9523
9524 ins_pipe(lmac_reg_reg);
9525 %}
9526
9527 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9528
9529 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9530 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9531
9532 ins_cost(INSN_COST * 3);
9533 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9534
9535 ins_encode %{
9536 __ smaddl(as_Register($dst$$reg),
9537 as_Register($src1$$reg),
9538 as_Register($src2$$reg),
9539 as_Register($src3$$reg));
9540 %}
9541
9542 ins_pipe(imac_reg_reg);
9543 %}
9544
9545 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9546 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9547
9548 ins_cost(INSN_COST * 3);
9549 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9550
9551 ins_encode %{
9552 __ smsubl(as_Register($dst$$reg),
9553 as_Register($src1$$reg),
9554 as_Register($src2$$reg),
9555 as_Register($src3$$reg));
9556 %}
9557
9558 ins_pipe(imac_reg_reg);
9559 %}
9560
9561 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9562 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9563
9564 ins_cost(INSN_COST * 3);
9565 format %{ "smnegl $dst, $src1, $src2" %}
9566
9567 ins_encode %{
9568 __ smnegl(as_Register($dst$$reg),
9569 as_Register($src1$$reg),
9570 as_Register($src2$$reg));
9571 %}
9572
9573 ins_pipe(imac_reg_reg);
9574 %}
9575
9576 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9577
9578 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9579 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9580
9581 ins_cost(INSN_COST * 5);
9582 format %{ "mulw rscratch1, $src1, $src2\n\t"
9583 "maddw $dst, $src3, $src4, rscratch1" %}
9584
9585 ins_encode %{
9586 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9587 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9588
9589 ins_pipe(imac_reg_reg);
9590 %}
9591
9592 // Integer Divide
9593
9594 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9595 match(Set dst (DivI src1 src2));
9596
9597 ins_cost(INSN_COST * 19);
9598 format %{ "sdivw $dst, $src1, $src2" %}
9599
9600 ins_encode(aarch64_enc_divw(dst, src1, src2));
9601 ins_pipe(idiv_reg_reg);
9602 %}
9603
9604 // Long Divide
9605
9606 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9607 match(Set dst (DivL src1 src2));
9608
9609 ins_cost(INSN_COST * 35);
9610 format %{ "sdiv $dst, $src1, $src2" %}
9611
9612 ins_encode(aarch64_enc_div(dst, src1, src2));
9613 ins_pipe(ldiv_reg_reg);
9614 %}
9615
9616 // Integer Remainder
9617
9618 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9619 match(Set dst (ModI src1 src2));
9620
9621 ins_cost(INSN_COST * 22);
9622 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9623 "msubw $dst, rscratch1, $src2, $src1" %}
9624
9625 ins_encode(aarch64_enc_modw(dst, src1, src2));
9626 ins_pipe(idiv_reg_reg);
9627 %}
9628
9629 // Long Remainder
9630
9631 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9632 match(Set dst (ModL src1 src2));
9633
9634 ins_cost(INSN_COST * 38);
9635 format %{ "sdiv rscratch1, $src1, $src2\n"
9636 "msub $dst, rscratch1, $src2, $src1" %}
9637
9638 ins_encode(aarch64_enc_mod(dst, src1, src2));
9639 ins_pipe(ldiv_reg_reg);
9640 %}
9641
9642 // Unsigned Integer Divide
9643
9644 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9645 match(Set dst (UDivI src1 src2));
9646
9647 ins_cost(INSN_COST * 19);
9648 format %{ "udivw $dst, $src1, $src2" %}
9649
9650 ins_encode %{
9651 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9652 %}
9653
9654 ins_pipe(idiv_reg_reg);
9655 %}
9656
9657 // Unsigned Long Divide
9658
9659 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9660 match(Set dst (UDivL src1 src2));
9661
9662 ins_cost(INSN_COST * 35);
9663 format %{ "udiv $dst, $src1, $src2" %}
9664
9665 ins_encode %{
9666 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9667 %}
9668
9669 ins_pipe(ldiv_reg_reg);
9670 %}
9671
9672 // Unsigned Integer Remainder
9673
9674 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9675 match(Set dst (UModI src1 src2));
9676
9677 ins_cost(INSN_COST * 22);
9678 format %{ "udivw rscratch1, $src1, $src2\n\t"
9679 "msubw $dst, rscratch1, $src2, $src1" %}
9680
9681 ins_encode %{
9682 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9683 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9684 %}
9685
9686 ins_pipe(idiv_reg_reg);
9687 %}
9688
9689 // Unsigned Long Remainder
9690
9691 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9692 match(Set dst (UModL src1 src2));
9693
9694 ins_cost(INSN_COST * 38);
9695 format %{ "udiv rscratch1, $src1, $src2\n"
9696 "msub $dst, rscratch1, $src2, $src1" %}
9697
9698 ins_encode %{
9699 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9700 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9701 %}
9702
9703 ins_pipe(ldiv_reg_reg);
9704 %}
9705
9706 // Integer Shifts
9707
9708 // Shift Left Register
9709 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9710 match(Set dst (LShiftI src1 src2));
9711
9712 ins_cost(INSN_COST * 2);
9713 format %{ "lslvw $dst, $src1, $src2" %}
9714
9715 ins_encode %{
9716 __ lslvw(as_Register($dst$$reg),
9717 as_Register($src1$$reg),
9718 as_Register($src2$$reg));
9719 %}
9720
9721 ins_pipe(ialu_reg_reg_vshift);
9722 %}
9723
9724 // Shift Left Immediate
9725 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9726 match(Set dst (LShiftI src1 src2));
9727
9728 ins_cost(INSN_COST);
9729 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9730
9731 ins_encode %{
9732 __ lslw(as_Register($dst$$reg),
9733 as_Register($src1$$reg),
9734 $src2$$constant & 0x1f);
9735 %}
9736
9737 ins_pipe(ialu_reg_shift);
9738 %}
9739
9740 // Shift Right Logical Register
9741 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9742 match(Set dst (URShiftI src1 src2));
9743
9744 ins_cost(INSN_COST * 2);
9745 format %{ "lsrvw $dst, $src1, $src2" %}
9746
9747 ins_encode %{
9748 __ lsrvw(as_Register($dst$$reg),
9749 as_Register($src1$$reg),
9750 as_Register($src2$$reg));
9751 %}
9752
9753 ins_pipe(ialu_reg_reg_vshift);
9754 %}
9755
9756 // Shift Right Logical Immediate
9757 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9758 match(Set dst (URShiftI src1 src2));
9759
9760 ins_cost(INSN_COST);
9761 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9762
9763 ins_encode %{
9764 __ lsrw(as_Register($dst$$reg),
9765 as_Register($src1$$reg),
9766 $src2$$constant & 0x1f);
9767 %}
9768
9769 ins_pipe(ialu_reg_shift);
9770 %}
9771
9772 // Shift Right Arithmetic Register
9773 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9774 match(Set dst (RShiftI src1 src2));
9775
9776 ins_cost(INSN_COST * 2);
9777 format %{ "asrvw $dst, $src1, $src2" %}
9778
9779 ins_encode %{
9780 __ asrvw(as_Register($dst$$reg),
9781 as_Register($src1$$reg),
9782 as_Register($src2$$reg));
9783 %}
9784
9785 ins_pipe(ialu_reg_reg_vshift);
9786 %}
9787
9788 // Shift Right Arithmetic Immediate
9789 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9790 match(Set dst (RShiftI src1 src2));
9791
9792 ins_cost(INSN_COST);
9793 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9794
9795 ins_encode %{
9796 __ asrw(as_Register($dst$$reg),
9797 as_Register($src1$$reg),
9798 $src2$$constant & 0x1f);
9799 %}
9800
9801 ins_pipe(ialu_reg_shift);
9802 %}
9803
9804 // Combined Int Mask and Right Shift (using UBFM)
9805 // TODO
9806
9807 // Long Shifts
9808
9809 // Shift Left Register
9810 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9811 match(Set dst (LShiftL src1 src2));
9812
9813 ins_cost(INSN_COST * 2);
9814 format %{ "lslv $dst, $src1, $src2" %}
9815
9816 ins_encode %{
9817 __ lslv(as_Register($dst$$reg),
9818 as_Register($src1$$reg),
9819 as_Register($src2$$reg));
9820 %}
9821
9822 ins_pipe(ialu_reg_reg_vshift);
9823 %}
9824
9825 // Shift Left Immediate
9826 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9827 match(Set dst (LShiftL src1 src2));
9828
9829 ins_cost(INSN_COST);
9830 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9831
9832 ins_encode %{
9833 __ lsl(as_Register($dst$$reg),
9834 as_Register($src1$$reg),
9835 $src2$$constant & 0x3f);
9836 %}
9837
9838 ins_pipe(ialu_reg_shift);
9839 %}
9840
9841 // Shift Right Logical Register
9842 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9843 match(Set dst (URShiftL src1 src2));
9844
9845 ins_cost(INSN_COST * 2);
9846 format %{ "lsrv $dst, $src1, $src2" %}
9847
9848 ins_encode %{
9849 __ lsrv(as_Register($dst$$reg),
9850 as_Register($src1$$reg),
9851 as_Register($src2$$reg));
9852 %}
9853
9854 ins_pipe(ialu_reg_reg_vshift);
9855 %}
9856
9857 // Shift Right Logical Immediate
9858 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9859 match(Set dst (URShiftL src1 src2));
9860
9861 ins_cost(INSN_COST);
9862 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9863
9864 ins_encode %{
9865 __ lsr(as_Register($dst$$reg),
9866 as_Register($src1$$reg),
9867 $src2$$constant & 0x3f);
9868 %}
9869
9870 ins_pipe(ialu_reg_shift);
9871 %}
9872
9873 // A special-case pattern for card table stores.
9874 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9875 match(Set dst (URShiftL (CastP2X src1) src2));
9876
9877 ins_cost(INSN_COST);
9878 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9879
9880 ins_encode %{
9881 __ lsr(as_Register($dst$$reg),
9882 as_Register($src1$$reg),
9883 $src2$$constant & 0x3f);
9884 %}
9885
9886 ins_pipe(ialu_reg_shift);
9887 %}
9888
9889 // Shift Right Arithmetic Register
9890 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9891 match(Set dst (RShiftL src1 src2));
9892
9893 ins_cost(INSN_COST * 2);
9894 format %{ "asrv $dst, $src1, $src2" %}
9895
9896 ins_encode %{
9897 __ asrv(as_Register($dst$$reg),
9898 as_Register($src1$$reg),
9899 as_Register($src2$$reg));
9900 %}
9901
9902 ins_pipe(ialu_reg_reg_vshift);
9903 %}
9904
9905 // Shift Right Arithmetic Immediate
9906 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9907 match(Set dst (RShiftL src1 src2));
9908
9909 ins_cost(INSN_COST);
9910 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9911
9912 ins_encode %{
9913 __ asr(as_Register($dst$$reg),
9914 as_Register($src1$$reg),
9915 $src2$$constant & 0x3f);
9916 %}
9917
9918 ins_pipe(ialu_reg_shift);
9919 %}
9920
9921 // BEGIN This section of the file is automatically generated. Do not edit --------------
9922 // This section is generated from aarch64_ad.m4
9923
9924 // This pattern is automatically generated from aarch64_ad.m4
9925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9926 instruct regL_not_reg(iRegLNoSp dst,
9927 iRegL src1, immL_M1 m1,
9928 rFlagsReg cr) %{
9929 match(Set dst (XorL src1 m1));
9930 ins_cost(INSN_COST);
9931 format %{ "eon $dst, $src1, zr" %}
9932
9933 ins_encode %{
9934 __ eon(as_Register($dst$$reg),
9935 as_Register($src1$$reg),
9936 zr,
9937 Assembler::LSL, 0);
9938 %}
9939
9940 ins_pipe(ialu_reg);
9941 %}
9942
9943 // This pattern is automatically generated from aarch64_ad.m4
9944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9945 instruct regI_not_reg(iRegINoSp dst,
9946 iRegIorL2I src1, immI_M1 m1,
9947 rFlagsReg cr) %{
9948 match(Set dst (XorI src1 m1));
9949 ins_cost(INSN_COST);
9950 format %{ "eonw $dst, $src1, zr" %}
9951
9952 ins_encode %{
9953 __ eonw(as_Register($dst$$reg),
9954 as_Register($src1$$reg),
9955 zr,
9956 Assembler::LSL, 0);
9957 %}
9958
9959 ins_pipe(ialu_reg);
9960 %}
9961
9962 // This pattern is automatically generated from aarch64_ad.m4
9963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9964 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9965 immI0 zero, iRegIorL2I src1, immI src2) %{
9966 match(Set dst (SubI zero (URShiftI src1 src2)));
9967
9968 ins_cost(1.9 * INSN_COST);
9969 format %{ "negw $dst, $src1, LSR $src2" %}
9970
9971 ins_encode %{
9972 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9973 Assembler::LSR, $src2$$constant & 0x1f);
9974 %}
9975
9976 ins_pipe(ialu_reg_shift);
9977 %}
9978
9979 // This pattern is automatically generated from aarch64_ad.m4
9980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9981 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9982 immI0 zero, iRegIorL2I src1, immI src2) %{
9983 match(Set dst (SubI zero (RShiftI src1 src2)));
9984
9985 ins_cost(1.9 * INSN_COST);
9986 format %{ "negw $dst, $src1, ASR $src2" %}
9987
9988 ins_encode %{
9989 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9990 Assembler::ASR, $src2$$constant & 0x1f);
9991 %}
9992
9993 ins_pipe(ialu_reg_shift);
9994 %}
9995
9996 // This pattern is automatically generated from aarch64_ad.m4
9997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9998 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9999 immI0 zero, iRegIorL2I src1, immI src2) %{
10000 match(Set dst (SubI zero (LShiftI src1 src2)));
10001
10002 ins_cost(1.9 * INSN_COST);
10003 format %{ "negw $dst, $src1, LSL $src2" %}
10004
10005 ins_encode %{
10006 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10007 Assembler::LSL, $src2$$constant & 0x1f);
10008 %}
10009
10010 ins_pipe(ialu_reg_shift);
10011 %}
10012
10013 // This pattern is automatically generated from aarch64_ad.m4
10014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10015 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10016 immL0 zero, iRegL src1, immI src2) %{
10017 match(Set dst (SubL zero (URShiftL src1 src2)));
10018
10019 ins_cost(1.9 * INSN_COST);
10020 format %{ "neg $dst, $src1, LSR $src2" %}
10021
10022 ins_encode %{
10023 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10024 Assembler::LSR, $src2$$constant & 0x3f);
10025 %}
10026
10027 ins_pipe(ialu_reg_shift);
10028 %}
10029
10030 // This pattern is automatically generated from aarch64_ad.m4
10031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10032 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10033 immL0 zero, iRegL src1, immI src2) %{
10034 match(Set dst (SubL zero (RShiftL src1 src2)));
10035
10036 ins_cost(1.9 * INSN_COST);
10037 format %{ "neg $dst, $src1, ASR $src2" %}
10038
10039 ins_encode %{
10040 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10041 Assembler::ASR, $src2$$constant & 0x3f);
10042 %}
10043
10044 ins_pipe(ialu_reg_shift);
10045 %}
10046
10047 // This pattern is automatically generated from aarch64_ad.m4
10048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10049 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10050 immL0 zero, iRegL src1, immI src2) %{
10051 match(Set dst (SubL zero (LShiftL src1 src2)));
10052
10053 ins_cost(1.9 * INSN_COST);
10054 format %{ "neg $dst, $src1, LSL $src2" %}
10055
10056 ins_encode %{
10057 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10058 Assembler::LSL, $src2$$constant & 0x3f);
10059 %}
10060
10061 ins_pipe(ialu_reg_shift);
10062 %}
10063
10064 // This pattern is automatically generated from aarch64_ad.m4
10065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10066 instruct AndI_reg_not_reg(iRegINoSp dst,
10067 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10068 match(Set dst (AndI src1 (XorI src2 m1)));
10069 ins_cost(INSN_COST);
10070 format %{ "bicw $dst, $src1, $src2" %}
10071
10072 ins_encode %{
10073 __ bicw(as_Register($dst$$reg),
10074 as_Register($src1$$reg),
10075 as_Register($src2$$reg),
10076 Assembler::LSL, 0);
10077 %}
10078
10079 ins_pipe(ialu_reg_reg);
10080 %}
10081
10082 // This pattern is automatically generated from aarch64_ad.m4
10083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10084 instruct AndL_reg_not_reg(iRegLNoSp dst,
10085 iRegL src1, iRegL src2, immL_M1 m1) %{
10086 match(Set dst (AndL src1 (XorL src2 m1)));
10087 ins_cost(INSN_COST);
10088 format %{ "bic $dst, $src1, $src2" %}
10089
10090 ins_encode %{
10091 __ bic(as_Register($dst$$reg),
10092 as_Register($src1$$reg),
10093 as_Register($src2$$reg),
10094 Assembler::LSL, 0);
10095 %}
10096
10097 ins_pipe(ialu_reg_reg);
10098 %}
10099
10100 // This pattern is automatically generated from aarch64_ad.m4
10101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10102 instruct OrI_reg_not_reg(iRegINoSp dst,
10103 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10104 match(Set dst (OrI src1 (XorI src2 m1)));
10105 ins_cost(INSN_COST);
10106 format %{ "ornw $dst, $src1, $src2" %}
10107
10108 ins_encode %{
10109 __ ornw(as_Register($dst$$reg),
10110 as_Register($src1$$reg),
10111 as_Register($src2$$reg),
10112 Assembler::LSL, 0);
10113 %}
10114
10115 ins_pipe(ialu_reg_reg);
10116 %}
10117
10118 // This pattern is automatically generated from aarch64_ad.m4
10119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10120 instruct OrL_reg_not_reg(iRegLNoSp dst,
10121 iRegL src1, iRegL src2, immL_M1 m1) %{
10122 match(Set dst (OrL src1 (XorL src2 m1)));
10123 ins_cost(INSN_COST);
10124 format %{ "orn $dst, $src1, $src2" %}
10125
10126 ins_encode %{
10127 __ orn(as_Register($dst$$reg),
10128 as_Register($src1$$reg),
10129 as_Register($src2$$reg),
10130 Assembler::LSL, 0);
10131 %}
10132
10133 ins_pipe(ialu_reg_reg);
10134 %}
10135
10136 // This pattern is automatically generated from aarch64_ad.m4
10137 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10138 instruct XorI_reg_not_reg(iRegINoSp dst,
10139 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10140 match(Set dst (XorI m1 (XorI src2 src1)));
10141 ins_cost(INSN_COST);
10142 format %{ "eonw $dst, $src1, $src2" %}
10143
10144 ins_encode %{
10145 __ eonw(as_Register($dst$$reg),
10146 as_Register($src1$$reg),
10147 as_Register($src2$$reg),
10148 Assembler::LSL, 0);
10149 %}
10150
10151 ins_pipe(ialu_reg_reg);
10152 %}
10153
10154 // This pattern is automatically generated from aarch64_ad.m4
10155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10156 instruct XorL_reg_not_reg(iRegLNoSp dst,
10157 iRegL src1, iRegL src2, immL_M1 m1) %{
10158 match(Set dst (XorL m1 (XorL src2 src1)));
10159 ins_cost(INSN_COST);
10160 format %{ "eon $dst, $src1, $src2" %}
10161
10162 ins_encode %{
10163 __ eon(as_Register($dst$$reg),
10164 as_Register($src1$$reg),
10165 as_Register($src2$$reg),
10166 Assembler::LSL, 0);
10167 %}
10168
10169 ins_pipe(ialu_reg_reg);
10170 %}
10171
10172 // This pattern is automatically generated from aarch64_ad.m4
10173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10174 // val & (-1 ^ (val >>> shift)) ==> bicw
10175 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10176 iRegIorL2I src1, iRegIorL2I src2,
10177 immI src3, immI_M1 src4) %{
10178 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10179 ins_cost(1.9 * INSN_COST);
10180 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10181
10182 ins_encode %{
10183 __ bicw(as_Register($dst$$reg),
10184 as_Register($src1$$reg),
10185 as_Register($src2$$reg),
10186 Assembler::LSR,
10187 $src3$$constant & 0x1f);
10188 %}
10189
10190 ins_pipe(ialu_reg_reg_shift);
10191 %}
10192
10193 // This pattern is automatically generated from aarch64_ad.m4
10194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10195 // val & (-1 ^ (val >>> shift)) ==> bic
10196 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10197 iRegL src1, iRegL src2,
10198 immI src3, immL_M1 src4) %{
10199 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10200 ins_cost(1.9 * INSN_COST);
10201 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10202
10203 ins_encode %{
10204 __ bic(as_Register($dst$$reg),
10205 as_Register($src1$$reg),
10206 as_Register($src2$$reg),
10207 Assembler::LSR,
10208 $src3$$constant & 0x3f);
10209 %}
10210
10211 ins_pipe(ialu_reg_reg_shift);
10212 %}
10213
10214 // This pattern is automatically generated from aarch64_ad.m4
10215 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10216 // val & (-1 ^ (val >> shift)) ==> bicw
10217 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10218 iRegIorL2I src1, iRegIorL2I src2,
10219 immI src3, immI_M1 src4) %{
10220 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10221 ins_cost(1.9 * INSN_COST);
10222 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10223
10224 ins_encode %{
10225 __ bicw(as_Register($dst$$reg),
10226 as_Register($src1$$reg),
10227 as_Register($src2$$reg),
10228 Assembler::ASR,
10229 $src3$$constant & 0x1f);
10230 %}
10231
10232 ins_pipe(ialu_reg_reg_shift);
10233 %}
10234
10235 // This pattern is automatically generated from aarch64_ad.m4
10236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10237 // val & (-1 ^ (val >> shift)) ==> bic
10238 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10239 iRegL src1, iRegL src2,
10240 immI src3, immL_M1 src4) %{
10241 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10242 ins_cost(1.9 * INSN_COST);
10243 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10244
10245 ins_encode %{
10246 __ bic(as_Register($dst$$reg),
10247 as_Register($src1$$reg),
10248 as_Register($src2$$reg),
10249 Assembler::ASR,
10250 $src3$$constant & 0x3f);
10251 %}
10252
10253 ins_pipe(ialu_reg_reg_shift);
10254 %}
10255
10256 // This pattern is automatically generated from aarch64_ad.m4
10257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10258 // val & (-1 ^ (val ror shift)) ==> bicw
10259 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10260 iRegIorL2I src1, iRegIorL2I src2,
10261 immI src3, immI_M1 src4) %{
10262 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10263 ins_cost(1.9 * INSN_COST);
10264 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10265
10266 ins_encode %{
10267 __ bicw(as_Register($dst$$reg),
10268 as_Register($src1$$reg),
10269 as_Register($src2$$reg),
10270 Assembler::ROR,
10271 $src3$$constant & 0x1f);
10272 %}
10273
10274 ins_pipe(ialu_reg_reg_shift);
10275 %}
10276
10277 // This pattern is automatically generated from aarch64_ad.m4
10278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10279 // val & (-1 ^ (val ror shift)) ==> bic
10280 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10281 iRegL src1, iRegL src2,
10282 immI src3, immL_M1 src4) %{
10283 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10284 ins_cost(1.9 * INSN_COST);
10285 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10286
10287 ins_encode %{
10288 __ bic(as_Register($dst$$reg),
10289 as_Register($src1$$reg),
10290 as_Register($src2$$reg),
10291 Assembler::ROR,
10292 $src3$$constant & 0x3f);
10293 %}
10294
10295 ins_pipe(ialu_reg_reg_shift);
10296 %}
10297
10298 // This pattern is automatically generated from aarch64_ad.m4
10299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10300 // val & (-1 ^ (val << shift)) ==> bicw
10301 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10302 iRegIorL2I src1, iRegIorL2I src2,
10303 immI src3, immI_M1 src4) %{
10304 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10305 ins_cost(1.9 * INSN_COST);
10306 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10307
10308 ins_encode %{
10309 __ bicw(as_Register($dst$$reg),
10310 as_Register($src1$$reg),
10311 as_Register($src2$$reg),
10312 Assembler::LSL,
10313 $src3$$constant & 0x1f);
10314 %}
10315
10316 ins_pipe(ialu_reg_reg_shift);
10317 %}
10318
10319 // This pattern is automatically generated from aarch64_ad.m4
10320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10321 // val & (-1 ^ (val << shift)) ==> bic
10322 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10323 iRegL src1, iRegL src2,
10324 immI src3, immL_M1 src4) %{
10325 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10326 ins_cost(1.9 * INSN_COST);
10327 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10328
10329 ins_encode %{
10330 __ bic(as_Register($dst$$reg),
10331 as_Register($src1$$reg),
10332 as_Register($src2$$reg),
10333 Assembler::LSL,
10334 $src3$$constant & 0x3f);
10335 %}
10336
10337 ins_pipe(ialu_reg_reg_shift);
10338 %}
10339
10340 // This pattern is automatically generated from aarch64_ad.m4
10341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10342 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10343 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10344 iRegIorL2I src1, iRegIorL2I src2,
10345 immI src3, immI_M1 src4) %{
10346 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10347 ins_cost(1.9 * INSN_COST);
10348 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10349
10350 ins_encode %{
10351 __ eonw(as_Register($dst$$reg),
10352 as_Register($src1$$reg),
10353 as_Register($src2$$reg),
10354 Assembler::LSR,
10355 $src3$$constant & 0x1f);
10356 %}
10357
10358 ins_pipe(ialu_reg_reg_shift);
10359 %}
10360
10361 // This pattern is automatically generated from aarch64_ad.m4
10362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10363 // val ^ (-1 ^ (val >>> shift)) ==> eon
10364 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10365 iRegL src1, iRegL src2,
10366 immI src3, immL_M1 src4) %{
10367 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10368 ins_cost(1.9 * INSN_COST);
10369 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10370
10371 ins_encode %{
10372 __ eon(as_Register($dst$$reg),
10373 as_Register($src1$$reg),
10374 as_Register($src2$$reg),
10375 Assembler::LSR,
10376 $src3$$constant & 0x3f);
10377 %}
10378
10379 ins_pipe(ialu_reg_reg_shift);
10380 %}
10381
10382 // This pattern is automatically generated from aarch64_ad.m4
10383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10384 // val ^ (-1 ^ (val >> shift)) ==> eonw
10385 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10386 iRegIorL2I src1, iRegIorL2I src2,
10387 immI src3, immI_M1 src4) %{
10388 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10389 ins_cost(1.9 * INSN_COST);
10390 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10391
10392 ins_encode %{
10393 __ eonw(as_Register($dst$$reg),
10394 as_Register($src1$$reg),
10395 as_Register($src2$$reg),
10396 Assembler::ASR,
10397 $src3$$constant & 0x1f);
10398 %}
10399
10400 ins_pipe(ialu_reg_reg_shift);
10401 %}
10402
10403 // This pattern is automatically generated from aarch64_ad.m4
10404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10405 // val ^ (-1 ^ (val >> shift)) ==> eon
10406 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10407 iRegL src1, iRegL src2,
10408 immI src3, immL_M1 src4) %{
10409 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10410 ins_cost(1.9 * INSN_COST);
10411 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10412
10413 ins_encode %{
10414 __ eon(as_Register($dst$$reg),
10415 as_Register($src1$$reg),
10416 as_Register($src2$$reg),
10417 Assembler::ASR,
10418 $src3$$constant & 0x3f);
10419 %}
10420
10421 ins_pipe(ialu_reg_reg_shift);
10422 %}
10423
10424 // This pattern is automatically generated from aarch64_ad.m4
10425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10426 // val ^ (-1 ^ (val ror shift)) ==> eonw
10427 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10428 iRegIorL2I src1, iRegIorL2I src2,
10429 immI src3, immI_M1 src4) %{
10430 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10431 ins_cost(1.9 * INSN_COST);
10432 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10433
10434 ins_encode %{
10435 __ eonw(as_Register($dst$$reg),
10436 as_Register($src1$$reg),
10437 as_Register($src2$$reg),
10438 Assembler::ROR,
10439 $src3$$constant & 0x1f);
10440 %}
10441
10442 ins_pipe(ialu_reg_reg_shift);
10443 %}
10444
10445 // This pattern is automatically generated from aarch64_ad.m4
10446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10447 // val ^ (-1 ^ (val ror shift)) ==> eon
10448 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10449 iRegL src1, iRegL src2,
10450 immI src3, immL_M1 src4) %{
10451 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10452 ins_cost(1.9 * INSN_COST);
10453 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10454
10455 ins_encode %{
10456 __ eon(as_Register($dst$$reg),
10457 as_Register($src1$$reg),
10458 as_Register($src2$$reg),
10459 Assembler::ROR,
10460 $src3$$constant & 0x3f);
10461 %}
10462
10463 ins_pipe(ialu_reg_reg_shift);
10464 %}
10465
10466 // This pattern is automatically generated from aarch64_ad.m4
10467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10468 // val ^ (-1 ^ (val << shift)) ==> eonw
10469 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10470 iRegIorL2I src1, iRegIorL2I src2,
10471 immI src3, immI_M1 src4) %{
10472 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10473 ins_cost(1.9 * INSN_COST);
10474 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10475
10476 ins_encode %{
10477 __ eonw(as_Register($dst$$reg),
10478 as_Register($src1$$reg),
10479 as_Register($src2$$reg),
10480 Assembler::LSL,
10481 $src3$$constant & 0x1f);
10482 %}
10483
10484 ins_pipe(ialu_reg_reg_shift);
10485 %}
10486
10487 // This pattern is automatically generated from aarch64_ad.m4
10488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10489 // val ^ (-1 ^ (val << shift)) ==> eon
10490 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10491 iRegL src1, iRegL src2,
10492 immI src3, immL_M1 src4) %{
10493 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10494 ins_cost(1.9 * INSN_COST);
10495 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10496
10497 ins_encode %{
10498 __ eon(as_Register($dst$$reg),
10499 as_Register($src1$$reg),
10500 as_Register($src2$$reg),
10501 Assembler::LSL,
10502 $src3$$constant & 0x3f);
10503 %}
10504
10505 ins_pipe(ialu_reg_reg_shift);
10506 %}
10507
10508 // This pattern is automatically generated from aarch64_ad.m4
10509 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10510 // val | (-1 ^ (val >>> shift)) ==> ornw
10511 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10512 iRegIorL2I src1, iRegIorL2I src2,
10513 immI src3, immI_M1 src4) %{
10514 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10515 ins_cost(1.9 * INSN_COST);
10516 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10517
10518 ins_encode %{
10519 __ ornw(as_Register($dst$$reg),
10520 as_Register($src1$$reg),
10521 as_Register($src2$$reg),
10522 Assembler::LSR,
10523 $src3$$constant & 0x1f);
10524 %}
10525
10526 ins_pipe(ialu_reg_reg_shift);
10527 %}
10528
10529 // This pattern is automatically generated from aarch64_ad.m4
10530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10531 // val | (-1 ^ (val >>> shift)) ==> orn
10532 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10533 iRegL src1, iRegL src2,
10534 immI src3, immL_M1 src4) %{
10535 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10536 ins_cost(1.9 * INSN_COST);
10537 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10538
10539 ins_encode %{
10540 __ orn(as_Register($dst$$reg),
10541 as_Register($src1$$reg),
10542 as_Register($src2$$reg),
10543 Assembler::LSR,
10544 $src3$$constant & 0x3f);
10545 %}
10546
10547 ins_pipe(ialu_reg_reg_shift);
10548 %}
10549
10550 // This pattern is automatically generated from aarch64_ad.m4
10551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10552 // val | (-1 ^ (val >> shift)) ==> ornw
10553 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10554 iRegIorL2I src1, iRegIorL2I src2,
10555 immI src3, immI_M1 src4) %{
10556 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10557 ins_cost(1.9 * INSN_COST);
10558 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10559
10560 ins_encode %{
10561 __ ornw(as_Register($dst$$reg),
10562 as_Register($src1$$reg),
10563 as_Register($src2$$reg),
10564 Assembler::ASR,
10565 $src3$$constant & 0x1f);
10566 %}
10567
10568 ins_pipe(ialu_reg_reg_shift);
10569 %}
10570
10571 // This pattern is automatically generated from aarch64_ad.m4
10572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10573 // val | (-1 ^ (val >> shift)) ==> orn
10574 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10575 iRegL src1, iRegL src2,
10576 immI src3, immL_M1 src4) %{
10577 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10578 ins_cost(1.9 * INSN_COST);
10579 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10580
10581 ins_encode %{
10582 __ orn(as_Register($dst$$reg),
10583 as_Register($src1$$reg),
10584 as_Register($src2$$reg),
10585 Assembler::ASR,
10586 $src3$$constant & 0x3f);
10587 %}
10588
10589 ins_pipe(ialu_reg_reg_shift);
10590 %}
10591
10592 // This pattern is automatically generated from aarch64_ad.m4
10593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10594 // val | (-1 ^ (val ror shift)) ==> ornw
10595 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10596 iRegIorL2I src1, iRegIorL2I src2,
10597 immI src3, immI_M1 src4) %{
10598 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10599 ins_cost(1.9 * INSN_COST);
10600 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10601
10602 ins_encode %{
10603 __ ornw(as_Register($dst$$reg),
10604 as_Register($src1$$reg),
10605 as_Register($src2$$reg),
10606 Assembler::ROR,
10607 $src3$$constant & 0x1f);
10608 %}
10609
10610 ins_pipe(ialu_reg_reg_shift);
10611 %}
10612
10613 // This pattern is automatically generated from aarch64_ad.m4
10614 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10615 // val | (-1 ^ (val ror shift)) ==> orn
10616 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10617 iRegL src1, iRegL src2,
10618 immI src3, immL_M1 src4) %{
10619 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10620 ins_cost(1.9 * INSN_COST);
10621 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10622
10623 ins_encode %{
10624 __ orn(as_Register($dst$$reg),
10625 as_Register($src1$$reg),
10626 as_Register($src2$$reg),
10627 Assembler::ROR,
10628 $src3$$constant & 0x3f);
10629 %}
10630
10631 ins_pipe(ialu_reg_reg_shift);
10632 %}
10633
10634 // This pattern is automatically generated from aarch64_ad.m4
10635 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10636 // val | (-1 ^ (val << shift)) ==> ornw
10637 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10638 iRegIorL2I src1, iRegIorL2I src2,
10639 immI src3, immI_M1 src4) %{
10640 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10641 ins_cost(1.9 * INSN_COST);
10642 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10643
10644 ins_encode %{
10645 __ ornw(as_Register($dst$$reg),
10646 as_Register($src1$$reg),
10647 as_Register($src2$$reg),
10648 Assembler::LSL,
10649 $src3$$constant & 0x1f);
10650 %}
10651
10652 ins_pipe(ialu_reg_reg_shift);
10653 %}
10654
10655 // This pattern is automatically generated from aarch64_ad.m4
10656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10657 // val | (-1 ^ (val << shift)) ==> orn
10658 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10659 iRegL src1, iRegL src2,
10660 immI src3, immL_M1 src4) %{
10661 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10662 ins_cost(1.9 * INSN_COST);
10663 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10664
10665 ins_encode %{
10666 __ orn(as_Register($dst$$reg),
10667 as_Register($src1$$reg),
10668 as_Register($src2$$reg),
10669 Assembler::LSL,
10670 $src3$$constant & 0x3f);
10671 %}
10672
10673 ins_pipe(ialu_reg_reg_shift);
10674 %}
10675
10676 // This pattern is automatically generated from aarch64_ad.m4
10677 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10678 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10679 iRegIorL2I src1, iRegIorL2I src2,
10680 immI src3) %{
10681 match(Set dst (AndI src1 (URShiftI src2 src3)));
10682
10683 ins_cost(1.9 * INSN_COST);
10684 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10685
10686 ins_encode %{
10687 __ andw(as_Register($dst$$reg),
10688 as_Register($src1$$reg),
10689 as_Register($src2$$reg),
10690 Assembler::LSR,
10691 $src3$$constant & 0x1f);
10692 %}
10693
10694 ins_pipe(ialu_reg_reg_shift);
10695 %}
10696
10697 // This pattern is automatically generated from aarch64_ad.m4
10698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10699 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10700 iRegL src1, iRegL src2,
10701 immI src3) %{
10702 match(Set dst (AndL src1 (URShiftL src2 src3)));
10703
10704 ins_cost(1.9 * INSN_COST);
10705 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10706
10707 ins_encode %{
10708 __ andr(as_Register($dst$$reg),
10709 as_Register($src1$$reg),
10710 as_Register($src2$$reg),
10711 Assembler::LSR,
10712 $src3$$constant & 0x3f);
10713 %}
10714
10715 ins_pipe(ialu_reg_reg_shift);
10716 %}
10717
10718 // This pattern is automatically generated from aarch64_ad.m4
10719 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10720 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10721 iRegIorL2I src1, iRegIorL2I src2,
10722 immI src3) %{
10723 match(Set dst (AndI src1 (RShiftI src2 src3)));
10724
10725 ins_cost(1.9 * INSN_COST);
10726 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10727
10728 ins_encode %{
10729 __ andw(as_Register($dst$$reg),
10730 as_Register($src1$$reg),
10731 as_Register($src2$$reg),
10732 Assembler::ASR,
10733 $src3$$constant & 0x1f);
10734 %}
10735
10736 ins_pipe(ialu_reg_reg_shift);
10737 %}
10738
10739 // This pattern is automatically generated from aarch64_ad.m4
10740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10741 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10742 iRegL src1, iRegL src2,
10743 immI src3) %{
10744 match(Set dst (AndL src1 (RShiftL src2 src3)));
10745
10746 ins_cost(1.9 * INSN_COST);
10747 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10748
10749 ins_encode %{
10750 __ andr(as_Register($dst$$reg),
10751 as_Register($src1$$reg),
10752 as_Register($src2$$reg),
10753 Assembler::ASR,
10754 $src3$$constant & 0x3f);
10755 %}
10756
10757 ins_pipe(ialu_reg_reg_shift);
10758 %}
10759
10760 // This pattern is automatically generated from aarch64_ad.m4
10761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10762 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10763 iRegIorL2I src1, iRegIorL2I src2,
10764 immI src3) %{
10765 match(Set dst (AndI src1 (LShiftI src2 src3)));
10766
10767 ins_cost(1.9 * INSN_COST);
10768 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10769
10770 ins_encode %{
10771 __ andw(as_Register($dst$$reg),
10772 as_Register($src1$$reg),
10773 as_Register($src2$$reg),
10774 Assembler::LSL,
10775 $src3$$constant & 0x1f);
10776 %}
10777
10778 ins_pipe(ialu_reg_reg_shift);
10779 %}
10780
10781 // This pattern is automatically generated from aarch64_ad.m4
10782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10783 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10784 iRegL src1, iRegL src2,
10785 immI src3) %{
10786 match(Set dst (AndL src1 (LShiftL src2 src3)));
10787
10788 ins_cost(1.9 * INSN_COST);
10789 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10790
10791 ins_encode %{
10792 __ andr(as_Register($dst$$reg),
10793 as_Register($src1$$reg),
10794 as_Register($src2$$reg),
10795 Assembler::LSL,
10796 $src3$$constant & 0x3f);
10797 %}
10798
10799 ins_pipe(ialu_reg_reg_shift);
10800 %}
10801
10802 // This pattern is automatically generated from aarch64_ad.m4
10803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10804 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10805 iRegIorL2I src1, iRegIorL2I src2,
10806 immI src3) %{
10807 match(Set dst (AndI src1 (RotateRight src2 src3)));
10808
10809 ins_cost(1.9 * INSN_COST);
10810 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10811
10812 ins_encode %{
10813 __ andw(as_Register($dst$$reg),
10814 as_Register($src1$$reg),
10815 as_Register($src2$$reg),
10816 Assembler::ROR,
10817 $src3$$constant & 0x1f);
10818 %}
10819
10820 ins_pipe(ialu_reg_reg_shift);
10821 %}
10822
10823 // This pattern is automatically generated from aarch64_ad.m4
10824 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10825 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10826 iRegL src1, iRegL src2,
10827 immI src3) %{
10828 match(Set dst (AndL src1 (RotateRight src2 src3)));
10829
10830 ins_cost(1.9 * INSN_COST);
10831 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10832
10833 ins_encode %{
10834 __ andr(as_Register($dst$$reg),
10835 as_Register($src1$$reg),
10836 as_Register($src2$$reg),
10837 Assembler::ROR,
10838 $src3$$constant & 0x3f);
10839 %}
10840
10841 ins_pipe(ialu_reg_reg_shift);
10842 %}
10843
10844 // This pattern is automatically generated from aarch64_ad.m4
10845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10846 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10847 iRegIorL2I src1, iRegIorL2I src2,
10848 immI src3) %{
10849 match(Set dst (XorI src1 (URShiftI src2 src3)));
10850
10851 ins_cost(1.9 * INSN_COST);
10852 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10853
10854 ins_encode %{
10855 __ eorw(as_Register($dst$$reg),
10856 as_Register($src1$$reg),
10857 as_Register($src2$$reg),
10858 Assembler::LSR,
10859 $src3$$constant & 0x1f);
10860 %}
10861
10862 ins_pipe(ialu_reg_reg_shift);
10863 %}
10864
10865 // This pattern is automatically generated from aarch64_ad.m4
10866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10867 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10868 iRegL src1, iRegL src2,
10869 immI src3) %{
10870 match(Set dst (XorL src1 (URShiftL src2 src3)));
10871
10872 ins_cost(1.9 * INSN_COST);
10873 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10874
10875 ins_encode %{
10876 __ eor(as_Register($dst$$reg),
10877 as_Register($src1$$reg),
10878 as_Register($src2$$reg),
10879 Assembler::LSR,
10880 $src3$$constant & 0x3f);
10881 %}
10882
10883 ins_pipe(ialu_reg_reg_shift);
10884 %}
10885
10886 // This pattern is automatically generated from aarch64_ad.m4
10887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10888 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10889 iRegIorL2I src1, iRegIorL2I src2,
10890 immI src3) %{
10891 match(Set dst (XorI src1 (RShiftI src2 src3)));
10892
10893 ins_cost(1.9 * INSN_COST);
10894 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10895
10896 ins_encode %{
10897 __ eorw(as_Register($dst$$reg),
10898 as_Register($src1$$reg),
10899 as_Register($src2$$reg),
10900 Assembler::ASR,
10901 $src3$$constant & 0x1f);
10902 %}
10903
10904 ins_pipe(ialu_reg_reg_shift);
10905 %}
10906
10907 // This pattern is automatically generated from aarch64_ad.m4
10908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10909 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10910 iRegL src1, iRegL src2,
10911 immI src3) %{
10912 match(Set dst (XorL src1 (RShiftL src2 src3)));
10913
10914 ins_cost(1.9 * INSN_COST);
10915 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10916
10917 ins_encode %{
10918 __ eor(as_Register($dst$$reg),
10919 as_Register($src1$$reg),
10920 as_Register($src2$$reg),
10921 Assembler::ASR,
10922 $src3$$constant & 0x3f);
10923 %}
10924
10925 ins_pipe(ialu_reg_reg_shift);
10926 %}
10927
10928 // This pattern is automatically generated from aarch64_ad.m4
10929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10930 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10931 iRegIorL2I src1, iRegIorL2I src2,
10932 immI src3) %{
10933 match(Set dst (XorI src1 (LShiftI src2 src3)));
10934
10935 ins_cost(1.9 * INSN_COST);
10936 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10937
10938 ins_encode %{
10939 __ eorw(as_Register($dst$$reg),
10940 as_Register($src1$$reg),
10941 as_Register($src2$$reg),
10942 Assembler::LSL,
10943 $src3$$constant & 0x1f);
10944 %}
10945
10946 ins_pipe(ialu_reg_reg_shift);
10947 %}
10948
10949 // This pattern is automatically generated from aarch64_ad.m4
10950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10951 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10952 iRegL src1, iRegL src2,
10953 immI src3) %{
10954 match(Set dst (XorL src1 (LShiftL src2 src3)));
10955
10956 ins_cost(1.9 * INSN_COST);
10957 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10958
10959 ins_encode %{
10960 __ eor(as_Register($dst$$reg),
10961 as_Register($src1$$reg),
10962 as_Register($src2$$reg),
10963 Assembler::LSL,
10964 $src3$$constant & 0x3f);
10965 %}
10966
10967 ins_pipe(ialu_reg_reg_shift);
10968 %}
10969
10970 // This pattern is automatically generated from aarch64_ad.m4
10971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10972 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10973 iRegIorL2I src1, iRegIorL2I src2,
10974 immI src3) %{
10975 match(Set dst (XorI src1 (RotateRight src2 src3)));
10976
10977 ins_cost(1.9 * INSN_COST);
10978 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10979
10980 ins_encode %{
10981 __ eorw(as_Register($dst$$reg),
10982 as_Register($src1$$reg),
10983 as_Register($src2$$reg),
10984 Assembler::ROR,
10985 $src3$$constant & 0x1f);
10986 %}
10987
10988 ins_pipe(ialu_reg_reg_shift);
10989 %}
10990
10991 // This pattern is automatically generated from aarch64_ad.m4
10992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10993 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10994 iRegL src1, iRegL src2,
10995 immI src3) %{
10996 match(Set dst (XorL src1 (RotateRight src2 src3)));
10997
10998 ins_cost(1.9 * INSN_COST);
10999 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11000
11001 ins_encode %{
11002 __ eor(as_Register($dst$$reg),
11003 as_Register($src1$$reg),
11004 as_Register($src2$$reg),
11005 Assembler::ROR,
11006 $src3$$constant & 0x3f);
11007 %}
11008
11009 ins_pipe(ialu_reg_reg_shift);
11010 %}
11011
11012 // This pattern is automatically generated from aarch64_ad.m4
11013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11014 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11015 iRegIorL2I src1, iRegIorL2I src2,
11016 immI src3) %{
11017 match(Set dst (OrI src1 (URShiftI src2 src3)));
11018
11019 ins_cost(1.9 * INSN_COST);
11020 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11021
11022 ins_encode %{
11023 __ orrw(as_Register($dst$$reg),
11024 as_Register($src1$$reg),
11025 as_Register($src2$$reg),
11026 Assembler::LSR,
11027 $src3$$constant & 0x1f);
11028 %}
11029
11030 ins_pipe(ialu_reg_reg_shift);
11031 %}
11032
11033 // This pattern is automatically generated from aarch64_ad.m4
11034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11035 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11036 iRegL src1, iRegL src2,
11037 immI src3) %{
11038 match(Set dst (OrL src1 (URShiftL src2 src3)));
11039
11040 ins_cost(1.9 * INSN_COST);
11041 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11042
11043 ins_encode %{
11044 __ orr(as_Register($dst$$reg),
11045 as_Register($src1$$reg),
11046 as_Register($src2$$reg),
11047 Assembler::LSR,
11048 $src3$$constant & 0x3f);
11049 %}
11050
11051 ins_pipe(ialu_reg_reg_shift);
11052 %}
11053
11054 // This pattern is automatically generated from aarch64_ad.m4
11055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11056 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11057 iRegIorL2I src1, iRegIorL2I src2,
11058 immI src3) %{
11059 match(Set dst (OrI src1 (RShiftI src2 src3)));
11060
11061 ins_cost(1.9 * INSN_COST);
11062 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11063
11064 ins_encode %{
11065 __ orrw(as_Register($dst$$reg),
11066 as_Register($src1$$reg),
11067 as_Register($src2$$reg),
11068 Assembler::ASR,
11069 $src3$$constant & 0x1f);
11070 %}
11071
11072 ins_pipe(ialu_reg_reg_shift);
11073 %}
11074
11075 // This pattern is automatically generated from aarch64_ad.m4
11076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11077 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11078 iRegL src1, iRegL src2,
11079 immI src3) %{
11080 match(Set dst (OrL src1 (RShiftL src2 src3)));
11081
11082 ins_cost(1.9 * INSN_COST);
11083 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11084
11085 ins_encode %{
11086 __ orr(as_Register($dst$$reg),
11087 as_Register($src1$$reg),
11088 as_Register($src2$$reg),
11089 Assembler::ASR,
11090 $src3$$constant & 0x3f);
11091 %}
11092
11093 ins_pipe(ialu_reg_reg_shift);
11094 %}
11095
11096 // This pattern is automatically generated from aarch64_ad.m4
11097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11098 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11099 iRegIorL2I src1, iRegIorL2I src2,
11100 immI src3) %{
11101 match(Set dst (OrI src1 (LShiftI src2 src3)));
11102
11103 ins_cost(1.9 * INSN_COST);
11104 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11105
11106 ins_encode %{
11107 __ orrw(as_Register($dst$$reg),
11108 as_Register($src1$$reg),
11109 as_Register($src2$$reg),
11110 Assembler::LSL,
11111 $src3$$constant & 0x1f);
11112 %}
11113
11114 ins_pipe(ialu_reg_reg_shift);
11115 %}
11116
11117 // This pattern is automatically generated from aarch64_ad.m4
11118 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11119 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11120 iRegL src1, iRegL src2,
11121 immI src3) %{
11122 match(Set dst (OrL src1 (LShiftL src2 src3)));
11123
11124 ins_cost(1.9 * INSN_COST);
11125 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11126
11127 ins_encode %{
11128 __ orr(as_Register($dst$$reg),
11129 as_Register($src1$$reg),
11130 as_Register($src2$$reg),
11131 Assembler::LSL,
11132 $src3$$constant & 0x3f);
11133 %}
11134
11135 ins_pipe(ialu_reg_reg_shift);
11136 %}
11137
11138 // This pattern is automatically generated from aarch64_ad.m4
11139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11140 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11141 iRegIorL2I src1, iRegIorL2I src2,
11142 immI src3) %{
11143 match(Set dst (OrI src1 (RotateRight src2 src3)));
11144
11145 ins_cost(1.9 * INSN_COST);
11146 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11147
11148 ins_encode %{
11149 __ orrw(as_Register($dst$$reg),
11150 as_Register($src1$$reg),
11151 as_Register($src2$$reg),
11152 Assembler::ROR,
11153 $src3$$constant & 0x1f);
11154 %}
11155
11156 ins_pipe(ialu_reg_reg_shift);
11157 %}
11158
11159 // This pattern is automatically generated from aarch64_ad.m4
11160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11161 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11162 iRegL src1, iRegL src2,
11163 immI src3) %{
11164 match(Set dst (OrL src1 (RotateRight src2 src3)));
11165
11166 ins_cost(1.9 * INSN_COST);
11167 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11168
11169 ins_encode %{
11170 __ orr(as_Register($dst$$reg),
11171 as_Register($src1$$reg),
11172 as_Register($src2$$reg),
11173 Assembler::ROR,
11174 $src3$$constant & 0x3f);
11175 %}
11176
11177 ins_pipe(ialu_reg_reg_shift);
11178 %}
11179
11180 // This pattern is automatically generated from aarch64_ad.m4
11181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11182 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11183 iRegIorL2I src1, iRegIorL2I src2,
11184 immI src3) %{
11185 match(Set dst (AddI src1 (URShiftI src2 src3)));
11186
11187 ins_cost(1.9 * INSN_COST);
11188 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11189
11190 ins_encode %{
11191 __ addw(as_Register($dst$$reg),
11192 as_Register($src1$$reg),
11193 as_Register($src2$$reg),
11194 Assembler::LSR,
11195 $src3$$constant & 0x1f);
11196 %}
11197
11198 ins_pipe(ialu_reg_reg_shift);
11199 %}
11200
11201 // This pattern is automatically generated from aarch64_ad.m4
11202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11203 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11204 iRegL src1, iRegL src2,
11205 immI src3) %{
11206 match(Set dst (AddL src1 (URShiftL src2 src3)));
11207
11208 ins_cost(1.9 * INSN_COST);
11209 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11210
11211 ins_encode %{
11212 __ add(as_Register($dst$$reg),
11213 as_Register($src1$$reg),
11214 as_Register($src2$$reg),
11215 Assembler::LSR,
11216 $src3$$constant & 0x3f);
11217 %}
11218
11219 ins_pipe(ialu_reg_reg_shift);
11220 %}
11221
11222 // This pattern is automatically generated from aarch64_ad.m4
11223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11224 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11225 iRegIorL2I src1, iRegIorL2I src2,
11226 immI src3) %{
11227 match(Set dst (AddI src1 (RShiftI src2 src3)));
11228
11229 ins_cost(1.9 * INSN_COST);
11230 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11231
11232 ins_encode %{
11233 __ addw(as_Register($dst$$reg),
11234 as_Register($src1$$reg),
11235 as_Register($src2$$reg),
11236 Assembler::ASR,
11237 $src3$$constant & 0x1f);
11238 %}
11239
11240 ins_pipe(ialu_reg_reg_shift);
11241 %}
11242
11243 // This pattern is automatically generated from aarch64_ad.m4
11244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11245 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11246 iRegL src1, iRegL src2,
11247 immI src3) %{
11248 match(Set dst (AddL src1 (RShiftL src2 src3)));
11249
11250 ins_cost(1.9 * INSN_COST);
11251 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11252
11253 ins_encode %{
11254 __ add(as_Register($dst$$reg),
11255 as_Register($src1$$reg),
11256 as_Register($src2$$reg),
11257 Assembler::ASR,
11258 $src3$$constant & 0x3f);
11259 %}
11260
11261 ins_pipe(ialu_reg_reg_shift);
11262 %}
11263
11264 // This pattern is automatically generated from aarch64_ad.m4
11265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11266 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11267 iRegIorL2I src1, iRegIorL2I src2,
11268 immI src3) %{
11269 match(Set dst (AddI src1 (LShiftI src2 src3)));
11270
11271 ins_cost(1.9 * INSN_COST);
11272 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11273
11274 ins_encode %{
11275 __ addw(as_Register($dst$$reg),
11276 as_Register($src1$$reg),
11277 as_Register($src2$$reg),
11278 Assembler::LSL,
11279 $src3$$constant & 0x1f);
11280 %}
11281
11282 ins_pipe(ialu_reg_reg_shift);
11283 %}
11284
11285 // This pattern is automatically generated from aarch64_ad.m4
11286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11287 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11288 iRegL src1, iRegL src2,
11289 immI src3) %{
11290 match(Set dst (AddL src1 (LShiftL src2 src3)));
11291
11292 ins_cost(1.9 * INSN_COST);
11293 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11294
11295 ins_encode %{
11296 __ add(as_Register($dst$$reg),
11297 as_Register($src1$$reg),
11298 as_Register($src2$$reg),
11299 Assembler::LSL,
11300 $src3$$constant & 0x3f);
11301 %}
11302
11303 ins_pipe(ialu_reg_reg_shift);
11304 %}
11305
11306 // This pattern is automatically generated from aarch64_ad.m4
11307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11308 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11309 iRegIorL2I src1, iRegIorL2I src2,
11310 immI src3) %{
11311 match(Set dst (SubI src1 (URShiftI src2 src3)));
11312
11313 ins_cost(1.9 * INSN_COST);
11314 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11315
11316 ins_encode %{
11317 __ subw(as_Register($dst$$reg),
11318 as_Register($src1$$reg),
11319 as_Register($src2$$reg),
11320 Assembler::LSR,
11321 $src3$$constant & 0x1f);
11322 %}
11323
11324 ins_pipe(ialu_reg_reg_shift);
11325 %}
11326
11327 // This pattern is automatically generated from aarch64_ad.m4
11328 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11329 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11330 iRegL src1, iRegL src2,
11331 immI src3) %{
11332 match(Set dst (SubL src1 (URShiftL src2 src3)));
11333
11334 ins_cost(1.9 * INSN_COST);
11335 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11336
11337 ins_encode %{
11338 __ sub(as_Register($dst$$reg),
11339 as_Register($src1$$reg),
11340 as_Register($src2$$reg),
11341 Assembler::LSR,
11342 $src3$$constant & 0x3f);
11343 %}
11344
11345 ins_pipe(ialu_reg_reg_shift);
11346 %}
11347
11348 // This pattern is automatically generated from aarch64_ad.m4
11349 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11350 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11351 iRegIorL2I src1, iRegIorL2I src2,
11352 immI src3) %{
11353 match(Set dst (SubI src1 (RShiftI src2 src3)));
11354
11355 ins_cost(1.9 * INSN_COST);
11356 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11357
11358 ins_encode %{
11359 __ subw(as_Register($dst$$reg),
11360 as_Register($src1$$reg),
11361 as_Register($src2$$reg),
11362 Assembler::ASR,
11363 $src3$$constant & 0x1f);
11364 %}
11365
11366 ins_pipe(ialu_reg_reg_shift);
11367 %}
11368
11369 // This pattern is automatically generated from aarch64_ad.m4
11370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11371 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11372 iRegL src1, iRegL src2,
11373 immI src3) %{
11374 match(Set dst (SubL src1 (RShiftL src2 src3)));
11375
11376 ins_cost(1.9 * INSN_COST);
11377 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11378
11379 ins_encode %{
11380 __ sub(as_Register($dst$$reg),
11381 as_Register($src1$$reg),
11382 as_Register($src2$$reg),
11383 Assembler::ASR,
11384 $src3$$constant & 0x3f);
11385 %}
11386
11387 ins_pipe(ialu_reg_reg_shift);
11388 %}
11389
11390 // This pattern is automatically generated from aarch64_ad.m4
11391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11392 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11393 iRegIorL2I src1, iRegIorL2I src2,
11394 immI src3) %{
11395 match(Set dst (SubI src1 (LShiftI src2 src3)));
11396
11397 ins_cost(1.9 * INSN_COST);
11398 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11399
11400 ins_encode %{
11401 __ subw(as_Register($dst$$reg),
11402 as_Register($src1$$reg),
11403 as_Register($src2$$reg),
11404 Assembler::LSL,
11405 $src3$$constant & 0x1f);
11406 %}
11407
11408 ins_pipe(ialu_reg_reg_shift);
11409 %}
11410
11411 // This pattern is automatically generated from aarch64_ad.m4
11412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11413 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11414 iRegL src1, iRegL src2,
11415 immI src3) %{
11416 match(Set dst (SubL src1 (LShiftL src2 src3)));
11417
11418 ins_cost(1.9 * INSN_COST);
11419 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11420
11421 ins_encode %{
11422 __ sub(as_Register($dst$$reg),
11423 as_Register($src1$$reg),
11424 as_Register($src2$$reg),
11425 Assembler::LSL,
11426 $src3$$constant & 0x3f);
11427 %}
11428
11429 ins_pipe(ialu_reg_reg_shift);
11430 %}
11431
11432 // This pattern is automatically generated from aarch64_ad.m4
11433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11434
11435 // Shift Left followed by Shift Right.
11436 // This idiom is used by the compiler for the i2b bytecode etc.
11437 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11438 %{
11439 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11440 ins_cost(INSN_COST * 2);
11441 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11442 ins_encode %{
11443 int lshift = $lshift_count$$constant & 63;
11444 int rshift = $rshift_count$$constant & 63;
11445 int s = 63 - lshift;
11446 int r = (rshift - lshift) & 63;
11447 __ sbfm(as_Register($dst$$reg),
11448 as_Register($src$$reg),
11449 r, s);
11450 %}
11451
11452 ins_pipe(ialu_reg_shift);
11453 %}
11454
11455 // This pattern is automatically generated from aarch64_ad.m4
11456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11457
11458 // Shift Left followed by Shift Right.
11459 // This idiom is used by the compiler for the i2b bytecode etc.
11460 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11461 %{
11462 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11463 ins_cost(INSN_COST * 2);
11464 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11465 ins_encode %{
11466 int lshift = $lshift_count$$constant & 31;
11467 int rshift = $rshift_count$$constant & 31;
11468 int s = 31 - lshift;
11469 int r = (rshift - lshift) & 31;
11470 __ sbfmw(as_Register($dst$$reg),
11471 as_Register($src$$reg),
11472 r, s);
11473 %}
11474
11475 ins_pipe(ialu_reg_shift);
11476 %}
11477
11478 // This pattern is automatically generated from aarch64_ad.m4
11479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11480
11481 // Shift Left followed by Shift Right.
11482 // This idiom is used by the compiler for the i2b bytecode etc.
11483 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11484 %{
11485 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11486 ins_cost(INSN_COST * 2);
11487 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11488 ins_encode %{
11489 int lshift = $lshift_count$$constant & 63;
11490 int rshift = $rshift_count$$constant & 63;
11491 int s = 63 - lshift;
11492 int r = (rshift - lshift) & 63;
11493 __ ubfm(as_Register($dst$$reg),
11494 as_Register($src$$reg),
11495 r, s);
11496 %}
11497
11498 ins_pipe(ialu_reg_shift);
11499 %}
11500
11501 // This pattern is automatically generated from aarch64_ad.m4
11502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11503
11504 // Shift Left followed by Shift Right.
11505 // This idiom is used by the compiler for the i2b bytecode etc.
11506 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11507 %{
11508 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11509 ins_cost(INSN_COST * 2);
11510 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11511 ins_encode %{
11512 int lshift = $lshift_count$$constant & 31;
11513 int rshift = $rshift_count$$constant & 31;
11514 int s = 31 - lshift;
11515 int r = (rshift - lshift) & 31;
11516 __ ubfmw(as_Register($dst$$reg),
11517 as_Register($src$$reg),
11518 r, s);
11519 %}
11520
11521 ins_pipe(ialu_reg_shift);
11522 %}
11523
11524 // Bitfield extract with shift & mask
11525
11526 // This pattern is automatically generated from aarch64_ad.m4
11527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11528 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11529 %{
11530 match(Set dst (AndI (URShiftI src rshift) mask));
11531 // Make sure we are not going to exceed what ubfxw can do.
11532 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11533
11534 ins_cost(INSN_COST);
11535 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11536 ins_encode %{
11537 int rshift = $rshift$$constant & 31;
11538 intptr_t mask = $mask$$constant;
11539 int width = exact_log2(mask+1);
11540 __ ubfxw(as_Register($dst$$reg),
11541 as_Register($src$$reg), rshift, width);
11542 %}
11543 ins_pipe(ialu_reg_shift);
11544 %}
11545
11546 // This pattern is automatically generated from aarch64_ad.m4
11547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11548 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11549 %{
11550 match(Set dst (AndL (URShiftL src rshift) mask));
11551 // Make sure we are not going to exceed what ubfx can do.
11552 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11553
11554 ins_cost(INSN_COST);
11555 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11556 ins_encode %{
11557 int rshift = $rshift$$constant & 63;
11558 intptr_t mask = $mask$$constant;
11559 int width = exact_log2_long(mask+1);
11560 __ ubfx(as_Register($dst$$reg),
11561 as_Register($src$$reg), rshift, width);
11562 %}
11563 ins_pipe(ialu_reg_shift);
11564 %}
11565
11566
11567 // This pattern is automatically generated from aarch64_ad.m4
11568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11569
11570 // We can use ubfx when extending an And with a mask when we know mask
11571 // is positive. We know that because immI_bitmask guarantees it.
11572 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11573 %{
11574 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11575 // Make sure we are not going to exceed what ubfxw can do.
11576 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11577
11578 ins_cost(INSN_COST * 2);
11579 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11580 ins_encode %{
11581 int rshift = $rshift$$constant & 31;
11582 intptr_t mask = $mask$$constant;
11583 int width = exact_log2(mask+1);
11584 __ ubfx(as_Register($dst$$reg),
11585 as_Register($src$$reg), rshift, width);
11586 %}
11587 ins_pipe(ialu_reg_shift);
11588 %}
11589
11590
11591 // This pattern is automatically generated from aarch64_ad.m4
11592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11593
11594 // We can use ubfiz when masking by a positive number and then left shifting the result.
11595 // We know that the mask is positive because immI_bitmask guarantees it.
11596 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11597 %{
11598 match(Set dst (LShiftI (AndI src mask) lshift));
11599 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11600
11601 ins_cost(INSN_COST);
11602 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11603 ins_encode %{
11604 int lshift = $lshift$$constant & 31;
11605 intptr_t mask = $mask$$constant;
11606 int width = exact_log2(mask+1);
11607 __ ubfizw(as_Register($dst$$reg),
11608 as_Register($src$$reg), lshift, width);
11609 %}
11610 ins_pipe(ialu_reg_shift);
11611 %}
11612
11613 // This pattern is automatically generated from aarch64_ad.m4
11614 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11615
11616 // We can use ubfiz when masking by a positive number and then left shifting the result.
11617 // We know that the mask is positive because immL_bitmask guarantees it.
11618 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11619 %{
11620 match(Set dst (LShiftL (AndL src mask) lshift));
11621 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11622
11623 ins_cost(INSN_COST);
11624 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11625 ins_encode %{
11626 int lshift = $lshift$$constant & 63;
11627 intptr_t mask = $mask$$constant;
11628 int width = exact_log2_long(mask+1);
11629 __ ubfiz(as_Register($dst$$reg),
11630 as_Register($src$$reg), lshift, width);
11631 %}
11632 ins_pipe(ialu_reg_shift);
11633 %}
11634
11635 // This pattern is automatically generated from aarch64_ad.m4
11636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11637
11638 // We can use ubfiz when masking by a positive number and then left shifting the result.
11639 // We know that the mask is positive because immI_bitmask guarantees it.
11640 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11641 %{
11642 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11643 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11644
11645 ins_cost(INSN_COST);
11646 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11647 ins_encode %{
11648 int lshift = $lshift$$constant & 31;
11649 intptr_t mask = $mask$$constant;
11650 int width = exact_log2(mask+1);
11651 __ ubfizw(as_Register($dst$$reg),
11652 as_Register($src$$reg), lshift, width);
11653 %}
11654 ins_pipe(ialu_reg_shift);
11655 %}
11656
11657 // This pattern is automatically generated from aarch64_ad.m4
11658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11659
11660 // We can use ubfiz when masking by a positive number and then left shifting the result.
11661 // We know that the mask is positive because immL_bitmask guarantees it.
11662 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11663 %{
11664 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11665 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11666
11667 ins_cost(INSN_COST);
11668 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11669 ins_encode %{
11670 int lshift = $lshift$$constant & 63;
11671 intptr_t mask = $mask$$constant;
11672 int width = exact_log2_long(mask+1);
11673 __ ubfiz(as_Register($dst$$reg),
11674 as_Register($src$$reg), lshift, width);
11675 %}
11676 ins_pipe(ialu_reg_shift);
11677 %}
11678
11679
11680 // This pattern is automatically generated from aarch64_ad.m4
11681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11682
11683 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11684 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11685 %{
11686 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11687 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11688
11689 ins_cost(INSN_COST);
11690 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11691 ins_encode %{
11692 int lshift = $lshift$$constant & 63;
11693 intptr_t mask = $mask$$constant;
11694 int width = exact_log2(mask+1);
11695 __ ubfiz(as_Register($dst$$reg),
11696 as_Register($src$$reg), lshift, width);
11697 %}
11698 ins_pipe(ialu_reg_shift);
11699 %}
11700
11701 // This pattern is automatically generated from aarch64_ad.m4
11702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11703
11704 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11705 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11706 %{
11707 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11708 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11709
11710 ins_cost(INSN_COST);
11711 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11712 ins_encode %{
11713 int lshift = $lshift$$constant & 31;
11714 intptr_t mask = $mask$$constant;
11715 int width = exact_log2(mask+1);
11716 __ ubfiz(as_Register($dst$$reg),
11717 as_Register($src$$reg), lshift, width);
11718 %}
11719 ins_pipe(ialu_reg_shift);
11720 %}
11721
11722 // This pattern is automatically generated from aarch64_ad.m4
11723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11724
11725 // Can skip int2long conversions after AND with small bitmask
11726 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11727 %{
11728 match(Set dst (ConvI2L (AndI src msk)));
11729 ins_cost(INSN_COST);
11730 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11731 ins_encode %{
11732 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11733 %}
11734 ins_pipe(ialu_reg_shift);
11735 %}
11736
11737
11738 // Rotations
11739
11740 // This pattern is automatically generated from aarch64_ad.m4
11741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11742 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11743 %{
11744 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11745 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11746
11747 ins_cost(INSN_COST);
11748 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11749
11750 ins_encode %{
11751 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11752 $rshift$$constant & 63);
11753 %}
11754 ins_pipe(ialu_reg_reg_extr);
11755 %}
11756
11757
11758 // This pattern is automatically generated from aarch64_ad.m4
11759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11760 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11761 %{
11762 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11763 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11764
11765 ins_cost(INSN_COST);
11766 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11767
11768 ins_encode %{
11769 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11770 $rshift$$constant & 31);
11771 %}
11772 ins_pipe(ialu_reg_reg_extr);
11773 %}
11774
11775
11776 // This pattern is automatically generated from aarch64_ad.m4
11777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11778 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11779 %{
11780 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11781 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11782
11783 ins_cost(INSN_COST);
11784 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11785
11786 ins_encode %{
11787 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11788 $rshift$$constant & 63);
11789 %}
11790 ins_pipe(ialu_reg_reg_extr);
11791 %}
11792
11793
11794 // This pattern is automatically generated from aarch64_ad.m4
11795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11796 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11797 %{
11798 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11799 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11800
11801 ins_cost(INSN_COST);
11802 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11803
11804 ins_encode %{
11805 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11806 $rshift$$constant & 31);
11807 %}
11808 ins_pipe(ialu_reg_reg_extr);
11809 %}
11810
11811 // This pattern is automatically generated from aarch64_ad.m4
11812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11813 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11814 %{
11815 match(Set dst (RotateRight src shift));
11816
11817 ins_cost(INSN_COST);
11818 format %{ "ror $dst, $src, $shift" %}
11819
11820 ins_encode %{
11821 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11822 $shift$$constant & 0x1f);
11823 %}
11824 ins_pipe(ialu_reg_reg_vshift);
11825 %}
11826
11827 // This pattern is automatically generated from aarch64_ad.m4
11828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11829 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11830 %{
11831 match(Set dst (RotateRight src shift));
11832
11833 ins_cost(INSN_COST);
11834 format %{ "ror $dst, $src, $shift" %}
11835
11836 ins_encode %{
11837 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11838 $shift$$constant & 0x3f);
11839 %}
11840 ins_pipe(ialu_reg_reg_vshift);
11841 %}
11842
11843 // This pattern is automatically generated from aarch64_ad.m4
11844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11845 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11846 %{
11847 match(Set dst (RotateRight src shift));
11848
11849 ins_cost(INSN_COST);
11850 format %{ "ror $dst, $src, $shift" %}
11851
11852 ins_encode %{
11853 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11854 %}
11855 ins_pipe(ialu_reg_reg_vshift);
11856 %}
11857
11858 // This pattern is automatically generated from aarch64_ad.m4
11859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11860 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11861 %{
11862 match(Set dst (RotateRight src shift));
11863
11864 ins_cost(INSN_COST);
11865 format %{ "ror $dst, $src, $shift" %}
11866
11867 ins_encode %{
11868 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11869 %}
11870 ins_pipe(ialu_reg_reg_vshift);
11871 %}
11872
11873 // This pattern is automatically generated from aarch64_ad.m4
11874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11875 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11876 %{
11877 match(Set dst (RotateLeft src shift));
11878
11879 ins_cost(INSN_COST);
11880 format %{ "rol $dst, $src, $shift" %}
11881
11882 ins_encode %{
11883 __ subw(rscratch1, zr, as_Register($shift$$reg));
11884 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11885 %}
11886 ins_pipe(ialu_reg_reg_vshift);
11887 %}
11888
11889 // This pattern is automatically generated from aarch64_ad.m4
11890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11891 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11892 %{
11893 match(Set dst (RotateLeft src shift));
11894
11895 ins_cost(INSN_COST);
11896 format %{ "rol $dst, $src, $shift" %}
11897
11898 ins_encode %{
11899 __ subw(rscratch1, zr, as_Register($shift$$reg));
11900 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11901 %}
11902 ins_pipe(ialu_reg_reg_vshift);
11903 %}
11904
11905
11906 // Add/subtract (extended)
11907
11908 // This pattern is automatically generated from aarch64_ad.m4
11909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11910 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11911 %{
11912 match(Set dst (AddL src1 (ConvI2L src2)));
11913 ins_cost(INSN_COST);
11914 format %{ "add $dst, $src1, $src2, sxtw" %}
11915
11916 ins_encode %{
11917 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11918 as_Register($src2$$reg), ext::sxtw);
11919 %}
11920 ins_pipe(ialu_reg_reg);
11921 %}
11922
11923 // This pattern is automatically generated from aarch64_ad.m4
11924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11925 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11926 %{
11927 match(Set dst (SubL src1 (ConvI2L src2)));
11928 ins_cost(INSN_COST);
11929 format %{ "sub $dst, $src1, $src2, sxtw" %}
11930
11931 ins_encode %{
11932 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11933 as_Register($src2$$reg), ext::sxtw);
11934 %}
11935 ins_pipe(ialu_reg_reg);
11936 %}
11937
11938 // This pattern is automatically generated from aarch64_ad.m4
11939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11940 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11941 %{
11942 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11943 ins_cost(INSN_COST);
11944 format %{ "add $dst, $src1, $src2, sxth" %}
11945
11946 ins_encode %{
11947 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11948 as_Register($src2$$reg), ext::sxth);
11949 %}
11950 ins_pipe(ialu_reg_reg);
11951 %}
11952
11953 // This pattern is automatically generated from aarch64_ad.m4
11954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11955 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11956 %{
11957 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11958 ins_cost(INSN_COST);
11959 format %{ "add $dst, $src1, $src2, sxtb" %}
11960
11961 ins_encode %{
11962 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11963 as_Register($src2$$reg), ext::sxtb);
11964 %}
11965 ins_pipe(ialu_reg_reg);
11966 %}
11967
11968 // This pattern is automatically generated from aarch64_ad.m4
11969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11970 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11971 %{
11972 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11973 ins_cost(INSN_COST);
11974 format %{ "add $dst, $src1, $src2, uxtb" %}
11975
11976 ins_encode %{
11977 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11978 as_Register($src2$$reg), ext::uxtb);
11979 %}
11980 ins_pipe(ialu_reg_reg);
11981 %}
11982
11983 // This pattern is automatically generated from aarch64_ad.m4
11984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11985 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11986 %{
11987 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11988 ins_cost(INSN_COST);
11989 format %{ "add $dst, $src1, $src2, sxth" %}
11990
11991 ins_encode %{
11992 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11993 as_Register($src2$$reg), ext::sxth);
11994 %}
11995 ins_pipe(ialu_reg_reg);
11996 %}
11997
11998 // This pattern is automatically generated from aarch64_ad.m4
11999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12000 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12001 %{
12002 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12003 ins_cost(INSN_COST);
12004 format %{ "add $dst, $src1, $src2, sxtw" %}
12005
12006 ins_encode %{
12007 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12008 as_Register($src2$$reg), ext::sxtw);
12009 %}
12010 ins_pipe(ialu_reg_reg);
12011 %}
12012
12013 // This pattern is automatically generated from aarch64_ad.m4
12014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12015 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12016 %{
12017 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12018 ins_cost(INSN_COST);
12019 format %{ "add $dst, $src1, $src2, sxtb" %}
12020
12021 ins_encode %{
12022 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12023 as_Register($src2$$reg), ext::sxtb);
12024 %}
12025 ins_pipe(ialu_reg_reg);
12026 %}
12027
12028 // This pattern is automatically generated from aarch64_ad.m4
12029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12030 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12031 %{
12032 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12033 ins_cost(INSN_COST);
12034 format %{ "add $dst, $src1, $src2, uxtb" %}
12035
12036 ins_encode %{
12037 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12038 as_Register($src2$$reg), ext::uxtb);
12039 %}
12040 ins_pipe(ialu_reg_reg);
12041 %}
12042
12043 // This pattern is automatically generated from aarch64_ad.m4
12044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12045 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12046 %{
12047 match(Set dst (AddI src1 (AndI src2 mask)));
12048 ins_cost(INSN_COST);
12049 format %{ "addw $dst, $src1, $src2, uxtb" %}
12050
12051 ins_encode %{
12052 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12053 as_Register($src2$$reg), ext::uxtb);
12054 %}
12055 ins_pipe(ialu_reg_reg);
12056 %}
12057
12058 // This pattern is automatically generated from aarch64_ad.m4
12059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12060 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12061 %{
12062 match(Set dst (AddI src1 (AndI src2 mask)));
12063 ins_cost(INSN_COST);
12064 format %{ "addw $dst, $src1, $src2, uxth" %}
12065
12066 ins_encode %{
12067 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12068 as_Register($src2$$reg), ext::uxth);
12069 %}
12070 ins_pipe(ialu_reg_reg);
12071 %}
12072
12073 // This pattern is automatically generated from aarch64_ad.m4
12074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12075 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12076 %{
12077 match(Set dst (AddL src1 (AndL src2 mask)));
12078 ins_cost(INSN_COST);
12079 format %{ "add $dst, $src1, $src2, uxtb" %}
12080
12081 ins_encode %{
12082 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12083 as_Register($src2$$reg), ext::uxtb);
12084 %}
12085 ins_pipe(ialu_reg_reg);
12086 %}
12087
12088 // This pattern is automatically generated from aarch64_ad.m4
12089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12090 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12091 %{
12092 match(Set dst (AddL src1 (AndL src2 mask)));
12093 ins_cost(INSN_COST);
12094 format %{ "add $dst, $src1, $src2, uxth" %}
12095
12096 ins_encode %{
12097 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12098 as_Register($src2$$reg), ext::uxth);
12099 %}
12100 ins_pipe(ialu_reg_reg);
12101 %}
12102
12103 // This pattern is automatically generated from aarch64_ad.m4
12104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12105 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12106 %{
12107 match(Set dst (AddL src1 (AndL src2 mask)));
12108 ins_cost(INSN_COST);
12109 format %{ "add $dst, $src1, $src2, uxtw" %}
12110
12111 ins_encode %{
12112 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12113 as_Register($src2$$reg), ext::uxtw);
12114 %}
12115 ins_pipe(ialu_reg_reg);
12116 %}
12117
12118 // This pattern is automatically generated from aarch64_ad.m4
12119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12120 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12121 %{
12122 match(Set dst (SubI src1 (AndI src2 mask)));
12123 ins_cost(INSN_COST);
12124 format %{ "subw $dst, $src1, $src2, uxtb" %}
12125
12126 ins_encode %{
12127 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12128 as_Register($src2$$reg), ext::uxtb);
12129 %}
12130 ins_pipe(ialu_reg_reg);
12131 %}
12132
12133 // This pattern is automatically generated from aarch64_ad.m4
12134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12135 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12136 %{
12137 match(Set dst (SubI src1 (AndI src2 mask)));
12138 ins_cost(INSN_COST);
12139 format %{ "subw $dst, $src1, $src2, uxth" %}
12140
12141 ins_encode %{
12142 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12143 as_Register($src2$$reg), ext::uxth);
12144 %}
12145 ins_pipe(ialu_reg_reg);
12146 %}
12147
12148 // This pattern is automatically generated from aarch64_ad.m4
12149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12150 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12151 %{
12152 match(Set dst (SubL src1 (AndL src2 mask)));
12153 ins_cost(INSN_COST);
12154 format %{ "sub $dst, $src1, $src2, uxtb" %}
12155
12156 ins_encode %{
12157 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12158 as_Register($src2$$reg), ext::uxtb);
12159 %}
12160 ins_pipe(ialu_reg_reg);
12161 %}
12162
12163 // This pattern is automatically generated from aarch64_ad.m4
12164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12165 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12166 %{
12167 match(Set dst (SubL src1 (AndL src2 mask)));
12168 ins_cost(INSN_COST);
12169 format %{ "sub $dst, $src1, $src2, uxth" %}
12170
12171 ins_encode %{
12172 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12173 as_Register($src2$$reg), ext::uxth);
12174 %}
12175 ins_pipe(ialu_reg_reg);
12176 %}
12177
12178 // This pattern is automatically generated from aarch64_ad.m4
12179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12180 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12181 %{
12182 match(Set dst (SubL src1 (AndL src2 mask)));
12183 ins_cost(INSN_COST);
12184 format %{ "sub $dst, $src1, $src2, uxtw" %}
12185
12186 ins_encode %{
12187 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12188 as_Register($src2$$reg), ext::uxtw);
12189 %}
12190 ins_pipe(ialu_reg_reg);
12191 %}
12192
12193
12194 // This pattern is automatically generated from aarch64_ad.m4
12195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12196 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12197 %{
12198 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12199 ins_cost(1.9 * INSN_COST);
12200 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12201
12202 ins_encode %{
12203 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12204 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12205 %}
12206 ins_pipe(ialu_reg_reg_shift);
12207 %}
12208
12209 // This pattern is automatically generated from aarch64_ad.m4
12210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12211 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12212 %{
12213 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12214 ins_cost(1.9 * INSN_COST);
12215 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12216
12217 ins_encode %{
12218 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12219 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12220 %}
12221 ins_pipe(ialu_reg_reg_shift);
12222 %}
12223
12224 // This pattern is automatically generated from aarch64_ad.m4
12225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12226 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12227 %{
12228 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12229 ins_cost(1.9 * INSN_COST);
12230 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12231
12232 ins_encode %{
12233 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12234 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12235 %}
12236 ins_pipe(ialu_reg_reg_shift);
12237 %}
12238
12239 // This pattern is automatically generated from aarch64_ad.m4
12240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12241 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12242 %{
12243 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12244 ins_cost(1.9 * INSN_COST);
12245 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12246
12247 ins_encode %{
12248 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12249 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12250 %}
12251 ins_pipe(ialu_reg_reg_shift);
12252 %}
12253
12254 // This pattern is automatically generated from aarch64_ad.m4
12255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12256 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12257 %{
12258 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12259 ins_cost(1.9 * INSN_COST);
12260 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12261
12262 ins_encode %{
12263 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12264 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12265 %}
12266 ins_pipe(ialu_reg_reg_shift);
12267 %}
12268
12269 // This pattern is automatically generated from aarch64_ad.m4
12270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12271 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12272 %{
12273 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12274 ins_cost(1.9 * INSN_COST);
12275 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12276
12277 ins_encode %{
12278 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12279 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12280 %}
12281 ins_pipe(ialu_reg_reg_shift);
12282 %}
12283
12284 // This pattern is automatically generated from aarch64_ad.m4
12285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12286 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12287 %{
12288 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12289 ins_cost(1.9 * INSN_COST);
12290 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12291
12292 ins_encode %{
12293 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12294 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12295 %}
12296 ins_pipe(ialu_reg_reg_shift);
12297 %}
12298
12299 // This pattern is automatically generated from aarch64_ad.m4
12300 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12301 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12302 %{
12303 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12304 ins_cost(1.9 * INSN_COST);
12305 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12306
12307 ins_encode %{
12308 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12309 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12310 %}
12311 ins_pipe(ialu_reg_reg_shift);
12312 %}
12313
12314 // This pattern is automatically generated from aarch64_ad.m4
12315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12316 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12317 %{
12318 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12319 ins_cost(1.9 * INSN_COST);
12320 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12321
12322 ins_encode %{
12323 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12324 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12325 %}
12326 ins_pipe(ialu_reg_reg_shift);
12327 %}
12328
12329 // This pattern is automatically generated from aarch64_ad.m4
12330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12331 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12332 %{
12333 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12334 ins_cost(1.9 * INSN_COST);
12335 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12336
12337 ins_encode %{
12338 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12339 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12340 %}
12341 ins_pipe(ialu_reg_reg_shift);
12342 %}
12343
12344 // This pattern is automatically generated from aarch64_ad.m4
12345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12346 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12347 %{
12348 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12349 ins_cost(1.9 * INSN_COST);
12350 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12351
12352 ins_encode %{
12353 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12354 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12355 %}
12356 ins_pipe(ialu_reg_reg_shift);
12357 %}
12358
12359 // This pattern is automatically generated from aarch64_ad.m4
12360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12361 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12362 %{
12363 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12364 ins_cost(1.9 * INSN_COST);
12365 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12366
12367 ins_encode %{
12368 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12369 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12370 %}
12371 ins_pipe(ialu_reg_reg_shift);
12372 %}
12373
12374 // This pattern is automatically generated from aarch64_ad.m4
12375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12376 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12377 %{
12378 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12379 ins_cost(1.9 * INSN_COST);
12380 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12381
12382 ins_encode %{
12383 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12384 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12385 %}
12386 ins_pipe(ialu_reg_reg_shift);
12387 %}
12388
12389 // This pattern is automatically generated from aarch64_ad.m4
12390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12391 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12392 %{
12393 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12394 ins_cost(1.9 * INSN_COST);
12395 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12396
12397 ins_encode %{
12398 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12399 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12400 %}
12401 ins_pipe(ialu_reg_reg_shift);
12402 %}
12403
12404 // This pattern is automatically generated from aarch64_ad.m4
12405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12406 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12407 %{
12408 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12409 ins_cost(1.9 * INSN_COST);
12410 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12411
12412 ins_encode %{
12413 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12414 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12415 %}
12416 ins_pipe(ialu_reg_reg_shift);
12417 %}
12418
12419 // This pattern is automatically generated from aarch64_ad.m4
12420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12421 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12422 %{
12423 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12424 ins_cost(1.9 * INSN_COST);
12425 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12426
12427 ins_encode %{
12428 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12429 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12430 %}
12431 ins_pipe(ialu_reg_reg_shift);
12432 %}
12433
12434 // This pattern is automatically generated from aarch64_ad.m4
12435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12436 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12437 %{
12438 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12439 ins_cost(1.9 * INSN_COST);
12440 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12441
12442 ins_encode %{
12443 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12444 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12445 %}
12446 ins_pipe(ialu_reg_reg_shift);
12447 %}
12448
12449 // This pattern is automatically generated from aarch64_ad.m4
12450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12451 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12452 %{
12453 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12454 ins_cost(1.9 * INSN_COST);
12455 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12456
12457 ins_encode %{
12458 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12459 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12460 %}
12461 ins_pipe(ialu_reg_reg_shift);
12462 %}
12463
12464 // This pattern is automatically generated from aarch64_ad.m4
12465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12466 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12467 %{
12468 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12469 ins_cost(1.9 * INSN_COST);
12470 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12471
12472 ins_encode %{
12473 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12474 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12475 %}
12476 ins_pipe(ialu_reg_reg_shift);
12477 %}
12478
12479 // This pattern is automatically generated from aarch64_ad.m4
12480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12481 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12482 %{
12483 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12484 ins_cost(1.9 * INSN_COST);
12485 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12486
12487 ins_encode %{
12488 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12489 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12490 %}
12491 ins_pipe(ialu_reg_reg_shift);
12492 %}
12493
12494 // This pattern is automatically generated from aarch64_ad.m4
12495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12496 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12497 %{
12498 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12499 ins_cost(1.9 * INSN_COST);
12500 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12501
12502 ins_encode %{
12503 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12504 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12505 %}
12506 ins_pipe(ialu_reg_reg_shift);
12507 %}
12508
12509 // This pattern is automatically generated from aarch64_ad.m4
12510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12511 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12512 %{
12513 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12514 ins_cost(1.9 * INSN_COST);
12515 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12516
12517 ins_encode %{
12518 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12519 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12520 %}
12521 ins_pipe(ialu_reg_reg_shift);
12522 %}
12523
12524 // This pattern is automatically generated from aarch64_ad.m4
12525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12526 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12527 %{
12528 effect(DEF dst, USE src1, USE src2, USE cr);
12529 ins_cost(INSN_COST * 2);
12530 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12531
12532 ins_encode %{
12533 __ cselw($dst$$Register,
12534 $src1$$Register,
12535 $src2$$Register,
12536 Assembler::LT);
12537 %}
12538 ins_pipe(icond_reg_reg);
12539 %}
12540
12541 // This pattern is automatically generated from aarch64_ad.m4
12542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12543 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12544 %{
12545 effect(DEF dst, USE src1, USE src2, USE cr);
12546 ins_cost(INSN_COST * 2);
12547 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12548
12549 ins_encode %{
12550 __ cselw($dst$$Register,
12551 $src1$$Register,
12552 $src2$$Register,
12553 Assembler::GT);
12554 %}
12555 ins_pipe(icond_reg_reg);
12556 %}
12557
12558 // This pattern is automatically generated from aarch64_ad.m4
12559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12560 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12561 %{
12562 effect(DEF dst, USE src1, USE cr);
12563 ins_cost(INSN_COST * 2);
12564 format %{ "cselw $dst, $src1, zr lt\t" %}
12565
12566 ins_encode %{
12567 __ cselw($dst$$Register,
12568 $src1$$Register,
12569 zr,
12570 Assembler::LT);
12571 %}
12572 ins_pipe(icond_reg);
12573 %}
12574
12575 // This pattern is automatically generated from aarch64_ad.m4
12576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12577 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12578 %{
12579 effect(DEF dst, USE src1, USE cr);
12580 ins_cost(INSN_COST * 2);
12581 format %{ "cselw $dst, $src1, zr gt\t" %}
12582
12583 ins_encode %{
12584 __ cselw($dst$$Register,
12585 $src1$$Register,
12586 zr,
12587 Assembler::GT);
12588 %}
12589 ins_pipe(icond_reg);
12590 %}
12591
12592 // This pattern is automatically generated from aarch64_ad.m4
12593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12594 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12595 %{
12596 effect(DEF dst, USE src1, USE cr);
12597 ins_cost(INSN_COST * 2);
12598 format %{ "csincw $dst, $src1, zr le\t" %}
12599
12600 ins_encode %{
12601 __ csincw($dst$$Register,
12602 $src1$$Register,
12603 zr,
12604 Assembler::LE);
12605 %}
12606 ins_pipe(icond_reg);
12607 %}
12608
12609 // This pattern is automatically generated from aarch64_ad.m4
12610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12611 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12612 %{
12613 effect(DEF dst, USE src1, USE cr);
12614 ins_cost(INSN_COST * 2);
12615 format %{ "csincw $dst, $src1, zr gt\t" %}
12616
12617 ins_encode %{
12618 __ csincw($dst$$Register,
12619 $src1$$Register,
12620 zr,
12621 Assembler::GT);
12622 %}
12623 ins_pipe(icond_reg);
12624 %}
12625
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12629 %{
12630 effect(DEF dst, USE src1, USE cr);
12631 ins_cost(INSN_COST * 2);
12632 format %{ "csinvw $dst, $src1, zr lt\t" %}
12633
12634 ins_encode %{
12635 __ csinvw($dst$$Register,
12636 $src1$$Register,
12637 zr,
12638 Assembler::LT);
12639 %}
12640 ins_pipe(icond_reg);
12641 %}
12642
12643 // This pattern is automatically generated from aarch64_ad.m4
12644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12645 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12646 %{
12647 effect(DEF dst, USE src1, USE cr);
12648 ins_cost(INSN_COST * 2);
12649 format %{ "csinvw $dst, $src1, zr ge\t" %}
12650
12651 ins_encode %{
12652 __ csinvw($dst$$Register,
12653 $src1$$Register,
12654 zr,
12655 Assembler::GE);
12656 %}
12657 ins_pipe(icond_reg);
12658 %}
12659
12660 // This pattern is automatically generated from aarch64_ad.m4
12661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12662 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12663 %{
12664 match(Set dst (MinI src imm));
12665 ins_cost(INSN_COST * 3);
12666 expand %{
12667 rFlagsReg cr;
12668 compI_reg_imm0(cr, src);
12669 cmovI_reg_imm0_lt(dst, src, cr);
12670 %}
12671 %}
12672
12673 // This pattern is automatically generated from aarch64_ad.m4
12674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12675 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12676 %{
12677 match(Set dst (MinI imm src));
12678 ins_cost(INSN_COST * 3);
12679 expand %{
12680 rFlagsReg cr;
12681 compI_reg_imm0(cr, src);
12682 cmovI_reg_imm0_lt(dst, src, cr);
12683 %}
12684 %}
12685
12686 // This pattern is automatically generated from aarch64_ad.m4
12687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12688 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12689 %{
12690 match(Set dst (MinI src imm));
12691 ins_cost(INSN_COST * 3);
12692 expand %{
12693 rFlagsReg cr;
12694 compI_reg_imm0(cr, src);
12695 cmovI_reg_imm1_le(dst, src, cr);
12696 %}
12697 %}
12698
12699 // This pattern is automatically generated from aarch64_ad.m4
12700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12701 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12702 %{
12703 match(Set dst (MinI imm src));
12704 ins_cost(INSN_COST * 3);
12705 expand %{
12706 rFlagsReg cr;
12707 compI_reg_imm0(cr, src);
12708 cmovI_reg_imm1_le(dst, src, cr);
12709 %}
12710 %}
12711
12712 // This pattern is automatically generated from aarch64_ad.m4
12713 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12714 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12715 %{
12716 match(Set dst (MinI src imm));
12717 ins_cost(INSN_COST * 3);
12718 expand %{
12719 rFlagsReg cr;
12720 compI_reg_imm0(cr, src);
12721 cmovI_reg_immM1_lt(dst, src, cr);
12722 %}
12723 %}
12724
12725 // This pattern is automatically generated from aarch64_ad.m4
12726 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12727 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12728 %{
12729 match(Set dst (MinI imm src));
12730 ins_cost(INSN_COST * 3);
12731 expand %{
12732 rFlagsReg cr;
12733 compI_reg_imm0(cr, src);
12734 cmovI_reg_immM1_lt(dst, src, cr);
12735 %}
12736 %}
12737
12738 // This pattern is automatically generated from aarch64_ad.m4
12739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12740 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12741 %{
12742 match(Set dst (MaxI src imm));
12743 ins_cost(INSN_COST * 3);
12744 expand %{
12745 rFlagsReg cr;
12746 compI_reg_imm0(cr, src);
12747 cmovI_reg_imm0_gt(dst, src, cr);
12748 %}
12749 %}
12750
12751 // This pattern is automatically generated from aarch64_ad.m4
12752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12753 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12754 %{
12755 match(Set dst (MaxI imm src));
12756 ins_cost(INSN_COST * 3);
12757 expand %{
12758 rFlagsReg cr;
12759 compI_reg_imm0(cr, src);
12760 cmovI_reg_imm0_gt(dst, src, cr);
12761 %}
12762 %}
12763
12764 // This pattern is automatically generated from aarch64_ad.m4
12765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12766 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12767 %{
12768 match(Set dst (MaxI src imm));
12769 ins_cost(INSN_COST * 3);
12770 expand %{
12771 rFlagsReg cr;
12772 compI_reg_imm0(cr, src);
12773 cmovI_reg_imm1_gt(dst, src, cr);
12774 %}
12775 %}
12776
12777 // This pattern is automatically generated from aarch64_ad.m4
12778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12779 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12780 %{
12781 match(Set dst (MaxI imm src));
12782 ins_cost(INSN_COST * 3);
12783 expand %{
12784 rFlagsReg cr;
12785 compI_reg_imm0(cr, src);
12786 cmovI_reg_imm1_gt(dst, src, cr);
12787 %}
12788 %}
12789
12790 // This pattern is automatically generated from aarch64_ad.m4
12791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12792 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12793 %{
12794 match(Set dst (MaxI src imm));
12795 ins_cost(INSN_COST * 3);
12796 expand %{
12797 rFlagsReg cr;
12798 compI_reg_imm0(cr, src);
12799 cmovI_reg_immM1_ge(dst, src, cr);
12800 %}
12801 %}
12802
12803 // This pattern is automatically generated from aarch64_ad.m4
12804 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12805 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12806 %{
12807 match(Set dst (MaxI imm src));
12808 ins_cost(INSN_COST * 3);
12809 expand %{
12810 rFlagsReg cr;
12811 compI_reg_imm0(cr, src);
12812 cmovI_reg_immM1_ge(dst, src, cr);
12813 %}
12814 %}
12815
12816 // This pattern is automatically generated from aarch64_ad.m4
12817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12818 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12819 %{
12820 match(Set dst (ReverseI src));
12821 ins_cost(INSN_COST);
12822 format %{ "rbitw $dst, $src" %}
12823 ins_encode %{
12824 __ rbitw($dst$$Register, $src$$Register);
12825 %}
12826 ins_pipe(ialu_reg);
12827 %}
12828
12829 // This pattern is automatically generated from aarch64_ad.m4
12830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12831 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12832 %{
12833 match(Set dst (ReverseL src));
12834 ins_cost(INSN_COST);
12835 format %{ "rbit $dst, $src" %}
12836 ins_encode %{
12837 __ rbit($dst$$Register, $src$$Register);
12838 %}
12839 ins_pipe(ialu_reg);
12840 %}
12841
12842
12843 // END This section of the file is automatically generated. Do not edit --------------
12844
12845
12846 // ============================================================================
12847 // Floating Point Arithmetic Instructions
12848
12849 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12850 match(Set dst (AddHF src1 src2));
12851 format %{ "faddh $dst, $src1, $src2" %}
12852 ins_encode %{
12853 __ faddh($dst$$FloatRegister,
12854 $src1$$FloatRegister,
12855 $src2$$FloatRegister);
12856 %}
12857 ins_pipe(fp_dop_reg_reg_s);
12858 %}
12859
12860 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12861 match(Set dst (AddF src1 src2));
12862
12863 ins_cost(INSN_COST * 5);
12864 format %{ "fadds $dst, $src1, $src2" %}
12865
12866 ins_encode %{
12867 __ fadds(as_FloatRegister($dst$$reg),
12868 as_FloatRegister($src1$$reg),
12869 as_FloatRegister($src2$$reg));
12870 %}
12871
12872 ins_pipe(fp_dop_reg_reg_s);
12873 %}
12874
12875 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12876 match(Set dst (AddD src1 src2));
12877
12878 ins_cost(INSN_COST * 5);
12879 format %{ "faddd $dst, $src1, $src2" %}
12880
12881 ins_encode %{
12882 __ faddd(as_FloatRegister($dst$$reg),
12883 as_FloatRegister($src1$$reg),
12884 as_FloatRegister($src2$$reg));
12885 %}
12886
12887 ins_pipe(fp_dop_reg_reg_d);
12888 %}
12889
12890 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12891 match(Set dst (SubHF src1 src2));
12892 format %{ "fsubh $dst, $src1, $src2" %}
12893 ins_encode %{
12894 __ fsubh($dst$$FloatRegister,
12895 $src1$$FloatRegister,
12896 $src2$$FloatRegister);
12897 %}
12898 ins_pipe(fp_dop_reg_reg_s);
12899 %}
12900
12901 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12902 match(Set dst (SubF src1 src2));
12903
12904 ins_cost(INSN_COST * 5);
12905 format %{ "fsubs $dst, $src1, $src2" %}
12906
12907 ins_encode %{
12908 __ fsubs(as_FloatRegister($dst$$reg),
12909 as_FloatRegister($src1$$reg),
12910 as_FloatRegister($src2$$reg));
12911 %}
12912
12913 ins_pipe(fp_dop_reg_reg_s);
12914 %}
12915
12916 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12917 match(Set dst (SubD src1 src2));
12918
12919 ins_cost(INSN_COST * 5);
12920 format %{ "fsubd $dst, $src1, $src2" %}
12921
12922 ins_encode %{
12923 __ fsubd(as_FloatRegister($dst$$reg),
12924 as_FloatRegister($src1$$reg),
12925 as_FloatRegister($src2$$reg));
12926 %}
12927
12928 ins_pipe(fp_dop_reg_reg_d);
12929 %}
12930
12931 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12932 match(Set dst (MulHF src1 src2));
12933 format %{ "fmulh $dst, $src1, $src2" %}
12934 ins_encode %{
12935 __ fmulh($dst$$FloatRegister,
12936 $src1$$FloatRegister,
12937 $src2$$FloatRegister);
12938 %}
12939 ins_pipe(fp_dop_reg_reg_s);
12940 %}
12941
12942 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12943 match(Set dst (MulF src1 src2));
12944
12945 ins_cost(INSN_COST * 6);
12946 format %{ "fmuls $dst, $src1, $src2" %}
12947
12948 ins_encode %{
12949 __ fmuls(as_FloatRegister($dst$$reg),
12950 as_FloatRegister($src1$$reg),
12951 as_FloatRegister($src2$$reg));
12952 %}
12953
12954 ins_pipe(fp_dop_reg_reg_s);
12955 %}
12956
12957 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12958 match(Set dst (MulD src1 src2));
12959
12960 ins_cost(INSN_COST * 6);
12961 format %{ "fmuld $dst, $src1, $src2" %}
12962
12963 ins_encode %{
12964 __ fmuld(as_FloatRegister($dst$$reg),
12965 as_FloatRegister($src1$$reg),
12966 as_FloatRegister($src2$$reg));
12967 %}
12968
12969 ins_pipe(fp_dop_reg_reg_d);
12970 %}
12971
12972 // src1 * src2 + src3 (half-precision float)
12973 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12974 match(Set dst (FmaHF src3 (Binary src1 src2)));
12975 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12976 ins_encode %{
12977 assert(UseFMA, "Needs FMA instructions support.");
12978 __ fmaddh($dst$$FloatRegister,
12979 $src1$$FloatRegister,
12980 $src2$$FloatRegister,
12981 $src3$$FloatRegister);
12982 %}
12983 ins_pipe(pipe_class_default);
12984 %}
12985
12986 // src1 * src2 + src3
12987 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12988 match(Set dst (FmaF src3 (Binary src1 src2)));
12989
12990 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12991
12992 ins_encode %{
12993 assert(UseFMA, "Needs FMA instructions support.");
12994 __ fmadds(as_FloatRegister($dst$$reg),
12995 as_FloatRegister($src1$$reg),
12996 as_FloatRegister($src2$$reg),
12997 as_FloatRegister($src3$$reg));
12998 %}
12999
13000 ins_pipe(pipe_class_default);
13001 %}
13002
13003 // src1 * src2 + src3
13004 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13005 match(Set dst (FmaD src3 (Binary src1 src2)));
13006
13007 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13008
13009 ins_encode %{
13010 assert(UseFMA, "Needs FMA instructions support.");
13011 __ fmaddd(as_FloatRegister($dst$$reg),
13012 as_FloatRegister($src1$$reg),
13013 as_FloatRegister($src2$$reg),
13014 as_FloatRegister($src3$$reg));
13015 %}
13016
13017 ins_pipe(pipe_class_default);
13018 %}
13019
13020 // src1 * (-src2) + src3
13021 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13022 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13023 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13024
13025 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13026
13027 ins_encode %{
13028 assert(UseFMA, "Needs FMA instructions support.");
13029 __ fmsubs(as_FloatRegister($dst$$reg),
13030 as_FloatRegister($src1$$reg),
13031 as_FloatRegister($src2$$reg),
13032 as_FloatRegister($src3$$reg));
13033 %}
13034
13035 ins_pipe(pipe_class_default);
13036 %}
13037
13038 // src1 * (-src2) + src3
13039 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13040 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13041 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13042
13043 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13044
13045 ins_encode %{
13046 assert(UseFMA, "Needs FMA instructions support.");
13047 __ fmsubd(as_FloatRegister($dst$$reg),
13048 as_FloatRegister($src1$$reg),
13049 as_FloatRegister($src2$$reg),
13050 as_FloatRegister($src3$$reg));
13051 %}
13052
13053 ins_pipe(pipe_class_default);
13054 %}
13055
13056 // src1 * (-src2) - src3
13057 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13058 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13059 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13060
13061 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13062
13063 ins_encode %{
13064 assert(UseFMA, "Needs FMA instructions support.");
13065 __ fnmadds(as_FloatRegister($dst$$reg),
13066 as_FloatRegister($src1$$reg),
13067 as_FloatRegister($src2$$reg),
13068 as_FloatRegister($src3$$reg));
13069 %}
13070
13071 ins_pipe(pipe_class_default);
13072 %}
13073
13074 // src1 * (-src2) - src3
13075 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13076 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13077 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13078
13079 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13080
13081 ins_encode %{
13082 assert(UseFMA, "Needs FMA instructions support.");
13083 __ fnmaddd(as_FloatRegister($dst$$reg),
13084 as_FloatRegister($src1$$reg),
13085 as_FloatRegister($src2$$reg),
13086 as_FloatRegister($src3$$reg));
13087 %}
13088
13089 ins_pipe(pipe_class_default);
13090 %}
13091
13092 // src1 * src2 - src3
13093 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13094 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13095
13096 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13097
13098 ins_encode %{
13099 assert(UseFMA, "Needs FMA instructions support.");
13100 __ fnmsubs(as_FloatRegister($dst$$reg),
13101 as_FloatRegister($src1$$reg),
13102 as_FloatRegister($src2$$reg),
13103 as_FloatRegister($src3$$reg));
13104 %}
13105
13106 ins_pipe(pipe_class_default);
13107 %}
13108
13109 // src1 * src2 - src3
13110 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13111 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13112
13113 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13114
13115 ins_encode %{
13116 assert(UseFMA, "Needs FMA instructions support.");
13117 // n.b. insn name should be fnmsubd
13118 __ fnmsub(as_FloatRegister($dst$$reg),
13119 as_FloatRegister($src1$$reg),
13120 as_FloatRegister($src2$$reg),
13121 as_FloatRegister($src3$$reg));
13122 %}
13123
13124 ins_pipe(pipe_class_default);
13125 %}
13126
13127 // Math.max(HH)H (half-precision float)
13128 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13129 match(Set dst (MaxHF src1 src2));
13130 format %{ "fmaxh $dst, $src1, $src2" %}
13131 ins_encode %{
13132 __ fmaxh($dst$$FloatRegister,
13133 $src1$$FloatRegister,
13134 $src2$$FloatRegister);
13135 %}
13136 ins_pipe(fp_dop_reg_reg_s);
13137 %}
13138
13139 // Math.min(HH)H (half-precision float)
13140 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13141 match(Set dst (MinHF src1 src2));
13142 format %{ "fminh $dst, $src1, $src2" %}
13143 ins_encode %{
13144 __ fminh($dst$$FloatRegister,
13145 $src1$$FloatRegister,
13146 $src2$$FloatRegister);
13147 %}
13148 ins_pipe(fp_dop_reg_reg_s);
13149 %}
13150
13151 // Math.max(FF)F
13152 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13153 match(Set dst (MaxF src1 src2));
13154
13155 format %{ "fmaxs $dst, $src1, $src2" %}
13156 ins_encode %{
13157 __ fmaxs(as_FloatRegister($dst$$reg),
13158 as_FloatRegister($src1$$reg),
13159 as_FloatRegister($src2$$reg));
13160 %}
13161
13162 ins_pipe(fp_dop_reg_reg_s);
13163 %}
13164
13165 // Math.min(FF)F
13166 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13167 match(Set dst (MinF src1 src2));
13168
13169 format %{ "fmins $dst, $src1, $src2" %}
13170 ins_encode %{
13171 __ fmins(as_FloatRegister($dst$$reg),
13172 as_FloatRegister($src1$$reg),
13173 as_FloatRegister($src2$$reg));
13174 %}
13175
13176 ins_pipe(fp_dop_reg_reg_s);
13177 %}
13178
13179 // Math.max(DD)D
13180 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13181 match(Set dst (MaxD src1 src2));
13182
13183 format %{ "fmaxd $dst, $src1, $src2" %}
13184 ins_encode %{
13185 __ fmaxd(as_FloatRegister($dst$$reg),
13186 as_FloatRegister($src1$$reg),
13187 as_FloatRegister($src2$$reg));
13188 %}
13189
13190 ins_pipe(fp_dop_reg_reg_d);
13191 %}
13192
13193 // Math.min(DD)D
13194 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13195 match(Set dst (MinD src1 src2));
13196
13197 format %{ "fmind $dst, $src1, $src2" %}
13198 ins_encode %{
13199 __ fmind(as_FloatRegister($dst$$reg),
13200 as_FloatRegister($src1$$reg),
13201 as_FloatRegister($src2$$reg));
13202 %}
13203
13204 ins_pipe(fp_dop_reg_reg_d);
13205 %}
13206
13207 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13208 match(Set dst (DivHF src1 src2));
13209 format %{ "fdivh $dst, $src1, $src2" %}
13210 ins_encode %{
13211 __ fdivh($dst$$FloatRegister,
13212 $src1$$FloatRegister,
13213 $src2$$FloatRegister);
13214 %}
13215 ins_pipe(fp_div_s);
13216 %}
13217
13218 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13219 match(Set dst (DivF src1 src2));
13220
13221 ins_cost(INSN_COST * 18);
13222 format %{ "fdivs $dst, $src1, $src2" %}
13223
13224 ins_encode %{
13225 __ fdivs(as_FloatRegister($dst$$reg),
13226 as_FloatRegister($src1$$reg),
13227 as_FloatRegister($src2$$reg));
13228 %}
13229
13230 ins_pipe(fp_div_s);
13231 %}
13232
13233 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13234 match(Set dst (DivD src1 src2));
13235
13236 ins_cost(INSN_COST * 32);
13237 format %{ "fdivd $dst, $src1, $src2" %}
13238
13239 ins_encode %{
13240 __ fdivd(as_FloatRegister($dst$$reg),
13241 as_FloatRegister($src1$$reg),
13242 as_FloatRegister($src2$$reg));
13243 %}
13244
13245 ins_pipe(fp_div_d);
13246 %}
13247
13248 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13249 match(Set dst (NegF src));
13250
13251 ins_cost(INSN_COST * 3);
13252 format %{ "fneg $dst, $src" %}
13253
13254 ins_encode %{
13255 __ fnegs(as_FloatRegister($dst$$reg),
13256 as_FloatRegister($src$$reg));
13257 %}
13258
13259 ins_pipe(fp_uop_s);
13260 %}
13261
13262 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13263 match(Set dst (NegD src));
13264
13265 ins_cost(INSN_COST * 3);
13266 format %{ "fnegd $dst, $src" %}
13267
13268 ins_encode %{
13269 __ fnegd(as_FloatRegister($dst$$reg),
13270 as_FloatRegister($src$$reg));
13271 %}
13272
13273 ins_pipe(fp_uop_d);
13274 %}
13275
13276 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13277 %{
13278 match(Set dst (AbsI src));
13279
13280 effect(KILL cr);
13281 ins_cost(INSN_COST * 2);
13282 format %{ "cmpw $src, zr\n\t"
13283 "cnegw $dst, $src, Assembler::LT\t# int abs"
13284 %}
13285
13286 ins_encode %{
13287 __ cmpw(as_Register($src$$reg), zr);
13288 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13289 %}
13290 ins_pipe(pipe_class_default);
13291 %}
13292
13293 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13294 %{
13295 match(Set dst (AbsL src));
13296
13297 effect(KILL cr);
13298 ins_cost(INSN_COST * 2);
13299 format %{ "cmp $src, zr\n\t"
13300 "cneg $dst, $src, Assembler::LT\t# long abs"
13301 %}
13302
13303 ins_encode %{
13304 __ cmp(as_Register($src$$reg), zr);
13305 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13306 %}
13307 ins_pipe(pipe_class_default);
13308 %}
13309
13310 instruct absF_reg(vRegF dst, vRegF src) %{
13311 match(Set dst (AbsF src));
13312
13313 ins_cost(INSN_COST * 3);
13314 format %{ "fabss $dst, $src" %}
13315 ins_encode %{
13316 __ fabss(as_FloatRegister($dst$$reg),
13317 as_FloatRegister($src$$reg));
13318 %}
13319
13320 ins_pipe(fp_uop_s);
13321 %}
13322
13323 instruct absD_reg(vRegD dst, vRegD src) %{
13324 match(Set dst (AbsD src));
13325
13326 ins_cost(INSN_COST * 3);
13327 format %{ "fabsd $dst, $src" %}
13328 ins_encode %{
13329 __ fabsd(as_FloatRegister($dst$$reg),
13330 as_FloatRegister($src$$reg));
13331 %}
13332
13333 ins_pipe(fp_uop_d);
13334 %}
13335
13336 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13337 match(Set dst (AbsF (SubF src1 src2)));
13338
13339 ins_cost(INSN_COST * 3);
13340 format %{ "fabds $dst, $src1, $src2" %}
13341 ins_encode %{
13342 __ fabds(as_FloatRegister($dst$$reg),
13343 as_FloatRegister($src1$$reg),
13344 as_FloatRegister($src2$$reg));
13345 %}
13346
13347 ins_pipe(fp_uop_s);
13348 %}
13349
13350 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13351 match(Set dst (AbsD (SubD src1 src2)));
13352
13353 ins_cost(INSN_COST * 3);
13354 format %{ "fabdd $dst, $src1, $src2" %}
13355 ins_encode %{
13356 __ fabdd(as_FloatRegister($dst$$reg),
13357 as_FloatRegister($src1$$reg),
13358 as_FloatRegister($src2$$reg));
13359 %}
13360
13361 ins_pipe(fp_uop_d);
13362 %}
13363
13364 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13365 match(Set dst (SqrtD src));
13366
13367 ins_cost(INSN_COST * 50);
13368 format %{ "fsqrtd $dst, $src" %}
13369 ins_encode %{
13370 __ fsqrtd(as_FloatRegister($dst$$reg),
13371 as_FloatRegister($src$$reg));
13372 %}
13373
13374 ins_pipe(fp_div_s);
13375 %}
13376
13377 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13378 match(Set dst (SqrtF src));
13379
13380 ins_cost(INSN_COST * 50);
13381 format %{ "fsqrts $dst, $src" %}
13382 ins_encode %{
13383 __ fsqrts(as_FloatRegister($dst$$reg),
13384 as_FloatRegister($src$$reg));
13385 %}
13386
13387 ins_pipe(fp_div_d);
13388 %}
13389
13390 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13391 match(Set dst (SqrtHF src));
13392 format %{ "fsqrth $dst, $src" %}
13393 ins_encode %{
13394 __ fsqrth($dst$$FloatRegister,
13395 $src$$FloatRegister);
13396 %}
13397 ins_pipe(fp_div_s);
13398 %}
13399
13400 // Math.rint, floor, ceil
13401 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13402 match(Set dst (RoundDoubleMode src rmode));
13403 format %{ "frint $dst, $src, $rmode" %}
13404 ins_encode %{
13405 switch ($rmode$$constant) {
13406 case RoundDoubleModeNode::rmode_rint:
13407 __ frintnd(as_FloatRegister($dst$$reg),
13408 as_FloatRegister($src$$reg));
13409 break;
13410 case RoundDoubleModeNode::rmode_floor:
13411 __ frintmd(as_FloatRegister($dst$$reg),
13412 as_FloatRegister($src$$reg));
13413 break;
13414 case RoundDoubleModeNode::rmode_ceil:
13415 __ frintpd(as_FloatRegister($dst$$reg),
13416 as_FloatRegister($src$$reg));
13417 break;
13418 }
13419 %}
13420 ins_pipe(fp_uop_d);
13421 %}
13422
13423 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13424 match(Set dst (CopySignD src1 (Binary src2 zero)));
13425 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13426 format %{ "CopySignD $dst $src1 $src2" %}
13427 ins_encode %{
13428 FloatRegister dst = as_FloatRegister($dst$$reg),
13429 src1 = as_FloatRegister($src1$$reg),
13430 src2 = as_FloatRegister($src2$$reg),
13431 zero = as_FloatRegister($zero$$reg);
13432 __ fnegd(dst, zero);
13433 __ bsl(dst, __ T8B, src2, src1);
13434 %}
13435 ins_pipe(fp_uop_d);
13436 %}
13437
13438 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13439 match(Set dst (CopySignF src1 src2));
13440 effect(TEMP_DEF dst, USE src1, USE src2);
13441 format %{ "CopySignF $dst $src1 $src2" %}
13442 ins_encode %{
13443 FloatRegister dst = as_FloatRegister($dst$$reg),
13444 src1 = as_FloatRegister($src1$$reg),
13445 src2 = as_FloatRegister($src2$$reg);
13446 __ movi(dst, __ T2S, 0x80, 24);
13447 __ bsl(dst, __ T8B, src2, src1);
13448 %}
13449 ins_pipe(fp_uop_d);
13450 %}
13451
13452 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13453 match(Set dst (SignumD src (Binary zero one)));
13454 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13455 format %{ "signumD $dst, $src" %}
13456 ins_encode %{
13457 FloatRegister src = as_FloatRegister($src$$reg),
13458 dst = as_FloatRegister($dst$$reg),
13459 zero = as_FloatRegister($zero$$reg),
13460 one = as_FloatRegister($one$$reg);
13461 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13462 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13463 // Bit selection instruction gets bit from "one" for each enabled bit in
13464 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13465 // NaN the whole "src" will be copied because "dst" is zero. For all other
13466 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13467 // from "src", and all other bits are copied from 1.0.
13468 __ bsl(dst, __ T8B, one, src);
13469 %}
13470 ins_pipe(fp_uop_d);
13471 %}
13472
13473 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13474 match(Set dst (SignumF src (Binary zero one)));
13475 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13476 format %{ "signumF $dst, $src" %}
13477 ins_encode %{
13478 FloatRegister src = as_FloatRegister($src$$reg),
13479 dst = as_FloatRegister($dst$$reg),
13480 zero = as_FloatRegister($zero$$reg),
13481 one = as_FloatRegister($one$$reg);
13482 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13483 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13484 // Bit selection instruction gets bit from "one" for each enabled bit in
13485 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13486 // NaN the whole "src" will be copied because "dst" is zero. For all other
13487 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13488 // from "src", and all other bits are copied from 1.0.
13489 __ bsl(dst, __ T8B, one, src);
13490 %}
13491 ins_pipe(fp_uop_d);
13492 %}
13493
13494 instruct onspinwait() %{
13495 match(OnSpinWait);
13496 ins_cost(INSN_COST);
13497
13498 format %{ "onspinwait" %}
13499
13500 ins_encode %{
13501 __ spin_wait();
13502 %}
13503 ins_pipe(pipe_class_empty);
13504 %}
13505
13506 // ============================================================================
13507 // Logical Instructions
13508
13509 // Integer Logical Instructions
13510
13511 // And Instructions
13512
13513
13514 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13515 match(Set dst (AndI src1 src2));
13516
13517 format %{ "andw $dst, $src1, $src2\t# int" %}
13518
13519 ins_cost(INSN_COST);
13520 ins_encode %{
13521 __ andw(as_Register($dst$$reg),
13522 as_Register($src1$$reg),
13523 as_Register($src2$$reg));
13524 %}
13525
13526 ins_pipe(ialu_reg_reg);
13527 %}
13528
13529 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13530 match(Set dst (AndI src1 src2));
13531
13532 format %{ "andsw $dst, $src1, $src2\t# int" %}
13533
13534 ins_cost(INSN_COST);
13535 ins_encode %{
13536 __ andw(as_Register($dst$$reg),
13537 as_Register($src1$$reg),
13538 (uint64_t)($src2$$constant));
13539 %}
13540
13541 ins_pipe(ialu_reg_imm);
13542 %}
13543
13544 // Or Instructions
13545
13546 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13547 match(Set dst (OrI src1 src2));
13548
13549 format %{ "orrw $dst, $src1, $src2\t# int" %}
13550
13551 ins_cost(INSN_COST);
13552 ins_encode %{
13553 __ orrw(as_Register($dst$$reg),
13554 as_Register($src1$$reg),
13555 as_Register($src2$$reg));
13556 %}
13557
13558 ins_pipe(ialu_reg_reg);
13559 %}
13560
13561 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13562 match(Set dst (OrI src1 src2));
13563
13564 format %{ "orrw $dst, $src1, $src2\t# int" %}
13565
13566 ins_cost(INSN_COST);
13567 ins_encode %{
13568 __ orrw(as_Register($dst$$reg),
13569 as_Register($src1$$reg),
13570 (uint64_t)($src2$$constant));
13571 %}
13572
13573 ins_pipe(ialu_reg_imm);
13574 %}
13575
13576 // Xor Instructions
13577
13578 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13579 match(Set dst (XorI src1 src2));
13580
13581 format %{ "eorw $dst, $src1, $src2\t# int" %}
13582
13583 ins_cost(INSN_COST);
13584 ins_encode %{
13585 __ eorw(as_Register($dst$$reg),
13586 as_Register($src1$$reg),
13587 as_Register($src2$$reg));
13588 %}
13589
13590 ins_pipe(ialu_reg_reg);
13591 %}
13592
13593 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13594 match(Set dst (XorI src1 src2));
13595
13596 format %{ "eorw $dst, $src1, $src2\t# int" %}
13597
13598 ins_cost(INSN_COST);
13599 ins_encode %{
13600 __ eorw(as_Register($dst$$reg),
13601 as_Register($src1$$reg),
13602 (uint64_t)($src2$$constant));
13603 %}
13604
13605 ins_pipe(ialu_reg_imm);
13606 %}
13607
13608 // Long Logical Instructions
13609 // TODO
13610
13611 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13612 match(Set dst (AndL src1 src2));
13613
13614 format %{ "and $dst, $src1, $src2\t# int" %}
13615
13616 ins_cost(INSN_COST);
13617 ins_encode %{
13618 __ andr(as_Register($dst$$reg),
13619 as_Register($src1$$reg),
13620 as_Register($src2$$reg));
13621 %}
13622
13623 ins_pipe(ialu_reg_reg);
13624 %}
13625
13626 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13627 match(Set dst (AndL src1 src2));
13628
13629 format %{ "and $dst, $src1, $src2\t# int" %}
13630
13631 ins_cost(INSN_COST);
13632 ins_encode %{
13633 __ andr(as_Register($dst$$reg),
13634 as_Register($src1$$reg),
13635 (uint64_t)($src2$$constant));
13636 %}
13637
13638 ins_pipe(ialu_reg_imm);
13639 %}
13640
13641 // Or Instructions
13642
13643 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13644 match(Set dst (OrL src1 src2));
13645
13646 format %{ "orr $dst, $src1, $src2\t# int" %}
13647
13648 ins_cost(INSN_COST);
13649 ins_encode %{
13650 __ orr(as_Register($dst$$reg),
13651 as_Register($src1$$reg),
13652 as_Register($src2$$reg));
13653 %}
13654
13655 ins_pipe(ialu_reg_reg);
13656 %}
13657
13658 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13659 match(Set dst (OrL src1 src2));
13660
13661 format %{ "orr $dst, $src1, $src2\t# int" %}
13662
13663 ins_cost(INSN_COST);
13664 ins_encode %{
13665 __ orr(as_Register($dst$$reg),
13666 as_Register($src1$$reg),
13667 (uint64_t)($src2$$constant));
13668 %}
13669
13670 ins_pipe(ialu_reg_imm);
13671 %}
13672
13673 // Xor Instructions
13674
13675 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13676 match(Set dst (XorL src1 src2));
13677
13678 format %{ "eor $dst, $src1, $src2\t# int" %}
13679
13680 ins_cost(INSN_COST);
13681 ins_encode %{
13682 __ eor(as_Register($dst$$reg),
13683 as_Register($src1$$reg),
13684 as_Register($src2$$reg));
13685 %}
13686
13687 ins_pipe(ialu_reg_reg);
13688 %}
13689
13690 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13691 match(Set dst (XorL src1 src2));
13692
13693 ins_cost(INSN_COST);
13694 format %{ "eor $dst, $src1, $src2\t# int" %}
13695
13696 ins_encode %{
13697 __ eor(as_Register($dst$$reg),
13698 as_Register($src1$$reg),
13699 (uint64_t)($src2$$constant));
13700 %}
13701
13702 ins_pipe(ialu_reg_imm);
13703 %}
13704
13705 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13706 %{
13707 match(Set dst (ConvI2L src));
13708
13709 ins_cost(INSN_COST);
13710 format %{ "sxtw $dst, $src\t# i2l" %}
13711 ins_encode %{
13712 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13713 %}
13714 ins_pipe(ialu_reg_shift);
13715 %}
13716
13717 // this pattern occurs in bigmath arithmetic
13718 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13719 %{
13720 match(Set dst (AndL (ConvI2L src) mask));
13721
13722 ins_cost(INSN_COST);
13723 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13724 ins_encode %{
13725 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13726 %}
13727
13728 ins_pipe(ialu_reg_shift);
13729 %}
13730
13731 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13732 match(Set dst (ConvL2I src));
13733
13734 ins_cost(INSN_COST);
13735 format %{ "movw $dst, $src \t// l2i" %}
13736
13737 ins_encode %{
13738 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13739 %}
13740
13741 ins_pipe(ialu_reg);
13742 %}
13743
13744 instruct convD2F_reg(vRegF dst, vRegD src) %{
13745 match(Set dst (ConvD2F src));
13746
13747 ins_cost(INSN_COST * 5);
13748 format %{ "fcvtd $dst, $src \t// d2f" %}
13749
13750 ins_encode %{
13751 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13752 %}
13753
13754 ins_pipe(fp_d2f);
13755 %}
13756
13757 instruct convF2D_reg(vRegD dst, vRegF src) %{
13758 match(Set dst (ConvF2D src));
13759
13760 ins_cost(INSN_COST * 5);
13761 format %{ "fcvts $dst, $src \t// f2d" %}
13762
13763 ins_encode %{
13764 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13765 %}
13766
13767 ins_pipe(fp_f2d);
13768 %}
13769
13770 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13771 match(Set dst (ConvF2I src));
13772
13773 ins_cost(INSN_COST * 5);
13774 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13775
13776 ins_encode %{
13777 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13778 %}
13779
13780 ins_pipe(fp_f2i);
13781 %}
13782
13783 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13784 match(Set dst (ConvF2L src));
13785
13786 ins_cost(INSN_COST * 5);
13787 format %{ "fcvtzs $dst, $src \t// f2l" %}
13788
13789 ins_encode %{
13790 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13791 %}
13792
13793 ins_pipe(fp_f2l);
13794 %}
13795
13796 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13797 match(Set dst (ConvF2HF src));
13798 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13799 "smov $dst, $tmp\t# move result from $tmp to $dst"
13800 %}
13801 effect(TEMP tmp);
13802 ins_encode %{
13803 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13804 %}
13805 ins_pipe(pipe_slow);
13806 %}
13807
13808 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13809 match(Set dst (ConvHF2F src));
13810 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13811 "fcvt $dst, $tmp\t# convert half to single precision"
13812 %}
13813 effect(TEMP tmp);
13814 ins_encode %{
13815 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13816 %}
13817 ins_pipe(pipe_slow);
13818 %}
13819
13820 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13821 match(Set dst (ConvI2F src));
13822
13823 ins_cost(INSN_COST * 5);
13824 format %{ "scvtfws $dst, $src \t// i2f" %}
13825
13826 ins_encode %{
13827 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13828 %}
13829
13830 ins_pipe(fp_i2f);
13831 %}
13832
13833 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13834 match(Set dst (ConvL2F src));
13835
13836 ins_cost(INSN_COST * 5);
13837 format %{ "scvtfs $dst, $src \t// l2f" %}
13838
13839 ins_encode %{
13840 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13841 %}
13842
13843 ins_pipe(fp_l2f);
13844 %}
13845
13846 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13847 match(Set dst (ConvD2I src));
13848
13849 ins_cost(INSN_COST * 5);
13850 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13851
13852 ins_encode %{
13853 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13854 %}
13855
13856 ins_pipe(fp_d2i);
13857 %}
13858
13859 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13860 match(Set dst (ConvD2L src));
13861
13862 ins_cost(INSN_COST * 5);
13863 format %{ "fcvtzd $dst, $src \t// d2l" %}
13864
13865 ins_encode %{
13866 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13867 %}
13868
13869 ins_pipe(fp_d2l);
13870 %}
13871
13872 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13873 match(Set dst (ConvI2D src));
13874
13875 ins_cost(INSN_COST * 5);
13876 format %{ "scvtfwd $dst, $src \t// i2d" %}
13877
13878 ins_encode %{
13879 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13880 %}
13881
13882 ins_pipe(fp_i2d);
13883 %}
13884
13885 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13886 match(Set dst (ConvL2D src));
13887
13888 ins_cost(INSN_COST * 5);
13889 format %{ "scvtfd $dst, $src \t// l2d" %}
13890
13891 ins_encode %{
13892 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13893 %}
13894
13895 ins_pipe(fp_l2d);
13896 %}
13897
13898 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13899 %{
13900 match(Set dst (RoundD src));
13901 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13902 format %{ "java_round_double $dst,$src"%}
13903 ins_encode %{
13904 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13905 as_FloatRegister($ftmp$$reg));
13906 %}
13907 ins_pipe(pipe_slow);
13908 %}
13909
13910 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13911 %{
13912 match(Set dst (RoundF src));
13913 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13914 format %{ "java_round_float $dst,$src"%}
13915 ins_encode %{
13916 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13917 as_FloatRegister($ftmp$$reg));
13918 %}
13919 ins_pipe(pipe_slow);
13920 %}
13921
13922 // stack <-> reg and reg <-> reg shuffles with no conversion
13923
13924 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13925
13926 match(Set dst (MoveF2I src));
13927
13928 effect(DEF dst, USE src);
13929
13930 ins_cost(4 * INSN_COST);
13931
13932 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13933
13934 ins_encode %{
13935 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13936 %}
13937
13938 ins_pipe(iload_reg_reg);
13939
13940 %}
13941
13942 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13943
13944 match(Set dst (MoveI2F src));
13945
13946 effect(DEF dst, USE src);
13947
13948 ins_cost(4 * INSN_COST);
13949
13950 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13951
13952 ins_encode %{
13953 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13954 %}
13955
13956 ins_pipe(pipe_class_memory);
13957
13958 %}
13959
13960 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13961
13962 match(Set dst (MoveD2L src));
13963
13964 effect(DEF dst, USE src);
13965
13966 ins_cost(4 * INSN_COST);
13967
13968 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13969
13970 ins_encode %{
13971 __ ldr($dst$$Register, Address(sp, $src$$disp));
13972 %}
13973
13974 ins_pipe(iload_reg_reg);
13975
13976 %}
13977
13978 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13979
13980 match(Set dst (MoveL2D src));
13981
13982 effect(DEF dst, USE src);
13983
13984 ins_cost(4 * INSN_COST);
13985
13986 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13987
13988 ins_encode %{
13989 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13990 %}
13991
13992 ins_pipe(pipe_class_memory);
13993
13994 %}
13995
13996 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13997
13998 match(Set dst (MoveF2I src));
13999
14000 effect(DEF dst, USE src);
14001
14002 ins_cost(INSN_COST);
14003
14004 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14005
14006 ins_encode %{
14007 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14008 %}
14009
14010 ins_pipe(pipe_class_memory);
14011
14012 %}
14013
14014 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14015
14016 match(Set dst (MoveI2F src));
14017
14018 effect(DEF dst, USE src);
14019
14020 ins_cost(INSN_COST);
14021
14022 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14023
14024 ins_encode %{
14025 __ strw($src$$Register, Address(sp, $dst$$disp));
14026 %}
14027
14028 ins_pipe(istore_reg_reg);
14029
14030 %}
14031
14032 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14033
14034 match(Set dst (MoveD2L src));
14035
14036 effect(DEF dst, USE src);
14037
14038 ins_cost(INSN_COST);
14039
14040 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14041
14042 ins_encode %{
14043 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14044 %}
14045
14046 ins_pipe(pipe_class_memory);
14047
14048 %}
14049
14050 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14051
14052 match(Set dst (MoveL2D src));
14053
14054 effect(DEF dst, USE src);
14055
14056 ins_cost(INSN_COST);
14057
14058 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14059
14060 ins_encode %{
14061 __ str($src$$Register, Address(sp, $dst$$disp));
14062 %}
14063
14064 ins_pipe(istore_reg_reg);
14065
14066 %}
14067
14068 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14069
14070 match(Set dst (MoveF2I src));
14071
14072 effect(DEF dst, USE src);
14073
14074 ins_cost(INSN_COST);
14075
14076 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14077
14078 ins_encode %{
14079 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14080 %}
14081
14082 ins_pipe(fp_f2i);
14083
14084 %}
14085
14086 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14087
14088 match(Set dst (MoveI2F src));
14089
14090 effect(DEF dst, USE src);
14091
14092 ins_cost(INSN_COST);
14093
14094 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14095
14096 ins_encode %{
14097 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14098 %}
14099
14100 ins_pipe(fp_i2f);
14101
14102 %}
14103
14104 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14105
14106 match(Set dst (MoveD2L src));
14107
14108 effect(DEF dst, USE src);
14109
14110 ins_cost(INSN_COST);
14111
14112 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14113
14114 ins_encode %{
14115 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14116 %}
14117
14118 ins_pipe(fp_d2l);
14119
14120 %}
14121
14122 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14123
14124 match(Set dst (MoveL2D src));
14125
14126 effect(DEF dst, USE src);
14127
14128 ins_cost(INSN_COST);
14129
14130 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14131
14132 ins_encode %{
14133 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14134 %}
14135
14136 ins_pipe(fp_l2d);
14137
14138 %}
14139
14140 // ============================================================================
14141 // clearing of an array
14142
14143 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14144 %{
14145 match(Set dummy (ClearArray (Binary cnt base) zero));
14146 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14147
14148 ins_cost(4 * INSN_COST);
14149 format %{ "ClearArray $cnt, $base" %}
14150
14151 ins_encode %{
14152 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14153 if (tpc == nullptr) {
14154 ciEnv::current()->record_failure("CodeCache is full");
14155 return;
14156 }
14157 %}
14158
14159 ins_pipe(pipe_class_memory);
14160 %}
14161
14162 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14163 %{
14164 predicate(((ClearArrayNode*)n)->word_copy_only());
14165 match(Set dummy (ClearArray (Binary cnt base) val));
14166 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14167
14168 ins_cost(4 * INSN_COST);
14169 format %{ "ClearArray $cnt, $base, $val" %}
14170
14171 ins_encode %{
14172 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14173 %}
14174
14175 ins_pipe(pipe_class_memory);
14176 %}
14177
14178 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14179 %{
14180 predicate((uint64_t)n->in(2)->get_long()
14181 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14182 && !((ClearArrayNode*)n)->word_copy_only());
14183 match(Set dummy (ClearArray cnt base));
14184 effect(TEMP temp, USE_KILL base, KILL cr);
14185
14186 ins_cost(4 * INSN_COST);
14187 format %{ "ClearArray $cnt, $base" %}
14188
14189 ins_encode %{
14190 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14191 if (tpc == nullptr) {
14192 ciEnv::current()->record_failure("CodeCache is full");
14193 return;
14194 }
14195 %}
14196
14197 ins_pipe(pipe_class_memory);
14198 %}
14199
14200 // ============================================================================
14201 // Overflow Math Instructions
14202
14203 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14204 %{
14205 match(Set cr (OverflowAddI op1 op2));
14206
14207 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14208 ins_cost(INSN_COST);
14209 ins_encode %{
14210 __ cmnw($op1$$Register, $op2$$Register);
14211 %}
14212
14213 ins_pipe(icmp_reg_reg);
14214 %}
14215
14216 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14217 %{
14218 match(Set cr (OverflowAddI op1 op2));
14219
14220 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14221 ins_cost(INSN_COST);
14222 ins_encode %{
14223 __ cmnw($op1$$Register, $op2$$constant);
14224 %}
14225
14226 ins_pipe(icmp_reg_imm);
14227 %}
14228
14229 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14230 %{
14231 match(Set cr (OverflowAddL op1 op2));
14232
14233 format %{ "cmn $op1, $op2\t# overflow check long" %}
14234 ins_cost(INSN_COST);
14235 ins_encode %{
14236 __ cmn($op1$$Register, $op2$$Register);
14237 %}
14238
14239 ins_pipe(icmp_reg_reg);
14240 %}
14241
14242 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14243 %{
14244 match(Set cr (OverflowAddL op1 op2));
14245
14246 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14247 ins_cost(INSN_COST);
14248 ins_encode %{
14249 __ adds(zr, $op1$$Register, $op2$$constant);
14250 %}
14251
14252 ins_pipe(icmp_reg_imm);
14253 %}
14254
14255 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14256 %{
14257 match(Set cr (OverflowSubI op1 op2));
14258
14259 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14260 ins_cost(INSN_COST);
14261 ins_encode %{
14262 __ cmpw($op1$$Register, $op2$$Register);
14263 %}
14264
14265 ins_pipe(icmp_reg_reg);
14266 %}
14267
14268 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14269 %{
14270 match(Set cr (OverflowSubI op1 op2));
14271
14272 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14273 ins_cost(INSN_COST);
14274 ins_encode %{
14275 __ cmpw($op1$$Register, $op2$$constant);
14276 %}
14277
14278 ins_pipe(icmp_reg_imm);
14279 %}
14280
14281 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14282 %{
14283 match(Set cr (OverflowSubL op1 op2));
14284
14285 format %{ "cmp $op1, $op2\t# overflow check long" %}
14286 ins_cost(INSN_COST);
14287 ins_encode %{
14288 __ cmp($op1$$Register, $op2$$Register);
14289 %}
14290
14291 ins_pipe(icmp_reg_reg);
14292 %}
14293
14294 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14295 %{
14296 match(Set cr (OverflowSubL op1 op2));
14297
14298 format %{ "cmp $op1, $op2\t# overflow check long" %}
14299 ins_cost(INSN_COST);
14300 ins_encode %{
14301 __ subs(zr, $op1$$Register, $op2$$constant);
14302 %}
14303
14304 ins_pipe(icmp_reg_imm);
14305 %}
14306
14307 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14308 %{
14309 match(Set cr (OverflowSubI zero op1));
14310
14311 format %{ "cmpw zr, $op1\t# overflow check int" %}
14312 ins_cost(INSN_COST);
14313 ins_encode %{
14314 __ cmpw(zr, $op1$$Register);
14315 %}
14316
14317 ins_pipe(icmp_reg_imm);
14318 %}
14319
14320 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14321 %{
14322 match(Set cr (OverflowSubL zero op1));
14323
14324 format %{ "cmp zr, $op1\t# overflow check long" %}
14325 ins_cost(INSN_COST);
14326 ins_encode %{
14327 __ cmp(zr, $op1$$Register);
14328 %}
14329
14330 ins_pipe(icmp_reg_imm);
14331 %}
14332
14333 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14334 %{
14335 match(Set cr (OverflowMulI op1 op2));
14336
14337 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14338 "cmp rscratch1, rscratch1, sxtw\n\t"
14339 "movw rscratch1, #0x80000000\n\t"
14340 "cselw rscratch1, rscratch1, zr, NE\n\t"
14341 "cmpw rscratch1, #1" %}
14342 ins_cost(5 * INSN_COST);
14343 ins_encode %{
14344 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14345 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14346 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14347 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14348 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14349 %}
14350
14351 ins_pipe(pipe_slow);
14352 %}
14353
14354 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14355 %{
14356 match(If cmp (OverflowMulI op1 op2));
14357 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14358 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14359 effect(USE labl, KILL cr);
14360
14361 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14362 "cmp rscratch1, rscratch1, sxtw\n\t"
14363 "b$cmp $labl" %}
14364 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14365 ins_encode %{
14366 Label* L = $labl$$label;
14367 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14368 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14369 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14370 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14371 %}
14372
14373 ins_pipe(pipe_serial);
14374 %}
14375
14376 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14377 %{
14378 match(Set cr (OverflowMulL op1 op2));
14379
14380 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14381 "smulh rscratch2, $op1, $op2\n\t"
14382 "cmp rscratch2, rscratch1, ASR #63\n\t"
14383 "movw rscratch1, #0x80000000\n\t"
14384 "cselw rscratch1, rscratch1, zr, NE\n\t"
14385 "cmpw rscratch1, #1" %}
14386 ins_cost(6 * INSN_COST);
14387 ins_encode %{
14388 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14389 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14390 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14391 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14392 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14393 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14394 %}
14395
14396 ins_pipe(pipe_slow);
14397 %}
14398
14399 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14400 %{
14401 match(If cmp (OverflowMulL op1 op2));
14402 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14403 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14404 effect(USE labl, KILL cr);
14405
14406 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14407 "smulh rscratch2, $op1, $op2\n\t"
14408 "cmp rscratch2, rscratch1, ASR #63\n\t"
14409 "b$cmp $labl" %}
14410 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14411 ins_encode %{
14412 Label* L = $labl$$label;
14413 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14414 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14415 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14416 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14417 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14418 %}
14419
14420 ins_pipe(pipe_serial);
14421 %}
14422
14423 // ============================================================================
14424 // Compare Instructions
14425
14426 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14427 %{
14428 match(Set cr (CmpI op1 op2));
14429
14430 effect(DEF cr, USE op1, USE op2);
14431
14432 ins_cost(INSN_COST);
14433 format %{ "cmpw $op1, $op2" %}
14434
14435 ins_encode(aarch64_enc_cmpw(op1, op2));
14436
14437 ins_pipe(icmp_reg_reg);
14438 %}
14439
14440 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14441 %{
14442 match(Set cr (CmpI op1 zero));
14443
14444 effect(DEF cr, USE op1);
14445
14446 ins_cost(INSN_COST);
14447 format %{ "cmpw $op1, 0" %}
14448
14449 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14450
14451 ins_pipe(icmp_reg_imm);
14452 %}
14453
14454 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14455 %{
14456 match(Set cr (CmpI op1 op2));
14457
14458 effect(DEF cr, USE op1);
14459
14460 ins_cost(INSN_COST);
14461 format %{ "cmpw $op1, $op2" %}
14462
14463 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14464
14465 ins_pipe(icmp_reg_imm);
14466 %}
14467
14468 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14469 %{
14470 match(Set cr (CmpI op1 op2));
14471
14472 effect(DEF cr, USE op1);
14473
14474 ins_cost(INSN_COST * 2);
14475 format %{ "cmpw $op1, $op2" %}
14476
14477 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14478
14479 ins_pipe(icmp_reg_imm);
14480 %}
14481
14482 // Unsigned compare Instructions; really, same as signed compare
14483 // except it should only be used to feed an If or a CMovI which takes a
14484 // cmpOpU.
14485
14486 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14487 %{
14488 match(Set cr (CmpU op1 op2));
14489
14490 effect(DEF cr, USE op1, USE op2);
14491
14492 ins_cost(INSN_COST);
14493 format %{ "cmpw $op1, $op2\t# unsigned" %}
14494
14495 ins_encode(aarch64_enc_cmpw(op1, op2));
14496
14497 ins_pipe(icmp_reg_reg);
14498 %}
14499
14500 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14501 %{
14502 match(Set cr (CmpU op1 zero));
14503
14504 effect(DEF cr, USE op1);
14505
14506 ins_cost(INSN_COST);
14507 format %{ "cmpw $op1, #0\t# unsigned" %}
14508
14509 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14510
14511 ins_pipe(icmp_reg_imm);
14512 %}
14513
14514 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14515 %{
14516 match(Set cr (CmpU op1 op2));
14517
14518 effect(DEF cr, USE op1);
14519
14520 ins_cost(INSN_COST);
14521 format %{ "cmpw $op1, $op2\t# unsigned" %}
14522
14523 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14524
14525 ins_pipe(icmp_reg_imm);
14526 %}
14527
14528 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14529 %{
14530 match(Set cr (CmpU op1 op2));
14531
14532 effect(DEF cr, USE op1);
14533
14534 ins_cost(INSN_COST * 2);
14535 format %{ "cmpw $op1, $op2\t# unsigned" %}
14536
14537 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14538
14539 ins_pipe(icmp_reg_imm);
14540 %}
14541
14542 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14543 %{
14544 match(Set cr (CmpL op1 op2));
14545
14546 effect(DEF cr, USE op1, USE op2);
14547
14548 ins_cost(INSN_COST);
14549 format %{ "cmp $op1, $op2" %}
14550
14551 ins_encode(aarch64_enc_cmp(op1, op2));
14552
14553 ins_pipe(icmp_reg_reg);
14554 %}
14555
14556 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14557 %{
14558 match(Set cr (CmpL op1 zero));
14559
14560 effect(DEF cr, USE op1);
14561
14562 ins_cost(INSN_COST);
14563 format %{ "tst $op1" %}
14564
14565 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14566
14567 ins_pipe(icmp_reg_imm);
14568 %}
14569
14570 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14571 %{
14572 match(Set cr (CmpL op1 op2));
14573
14574 effect(DEF cr, USE op1);
14575
14576 ins_cost(INSN_COST);
14577 format %{ "cmp $op1, $op2" %}
14578
14579 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14580
14581 ins_pipe(icmp_reg_imm);
14582 %}
14583
14584 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14585 %{
14586 match(Set cr (CmpL op1 op2));
14587
14588 effect(DEF cr, USE op1);
14589
14590 ins_cost(INSN_COST * 2);
14591 format %{ "cmp $op1, $op2" %}
14592
14593 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14594
14595 ins_pipe(icmp_reg_imm);
14596 %}
14597
14598 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14599 %{
14600 match(Set cr (CmpUL op1 op2));
14601
14602 effect(DEF cr, USE op1, USE op2);
14603
14604 ins_cost(INSN_COST);
14605 format %{ "cmp $op1, $op2" %}
14606
14607 ins_encode(aarch64_enc_cmp(op1, op2));
14608
14609 ins_pipe(icmp_reg_reg);
14610 %}
14611
14612 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14613 %{
14614 match(Set cr (CmpUL op1 zero));
14615
14616 effect(DEF cr, USE op1);
14617
14618 ins_cost(INSN_COST);
14619 format %{ "tst $op1" %}
14620
14621 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14622
14623 ins_pipe(icmp_reg_imm);
14624 %}
14625
14626 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14627 %{
14628 match(Set cr (CmpUL op1 op2));
14629
14630 effect(DEF cr, USE op1);
14631
14632 ins_cost(INSN_COST);
14633 format %{ "cmp $op1, $op2" %}
14634
14635 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14636
14637 ins_pipe(icmp_reg_imm);
14638 %}
14639
14640 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14641 %{
14642 match(Set cr (CmpUL op1 op2));
14643
14644 effect(DEF cr, USE op1);
14645
14646 ins_cost(INSN_COST * 2);
14647 format %{ "cmp $op1, $op2" %}
14648
14649 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14650
14651 ins_pipe(icmp_reg_imm);
14652 %}
14653
14654 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14655 %{
14656 match(Set cr (CmpP op1 op2));
14657
14658 effect(DEF cr, USE op1, USE op2);
14659
14660 ins_cost(INSN_COST);
14661 format %{ "cmp $op1, $op2\t // ptr" %}
14662
14663 ins_encode(aarch64_enc_cmpp(op1, op2));
14664
14665 ins_pipe(icmp_reg_reg);
14666 %}
14667
14668 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14669 %{
14670 match(Set cr (CmpN op1 op2));
14671
14672 effect(DEF cr, USE op1, USE op2);
14673
14674 ins_cost(INSN_COST);
14675 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14676
14677 ins_encode(aarch64_enc_cmpn(op1, op2));
14678
14679 ins_pipe(icmp_reg_reg);
14680 %}
14681
14682 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14683 %{
14684 match(Set cr (CmpP op1 zero));
14685
14686 effect(DEF cr, USE op1, USE zero);
14687
14688 ins_cost(INSN_COST);
14689 format %{ "cmp $op1, 0\t // ptr" %}
14690
14691 ins_encode(aarch64_enc_testp(op1));
14692
14693 ins_pipe(icmp_reg_imm);
14694 %}
14695
14696 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14697 %{
14698 match(Set cr (CmpN op1 zero));
14699
14700 effect(DEF cr, USE op1, USE zero);
14701
14702 ins_cost(INSN_COST);
14703 format %{ "cmp $op1, 0\t // compressed ptr" %}
14704
14705 ins_encode(aarch64_enc_testn(op1));
14706
14707 ins_pipe(icmp_reg_imm);
14708 %}
14709
14710 // FP comparisons
14711 //
14712 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14713 // using normal cmpOp. See declaration of rFlagsReg for details.
14714
14715 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14716 %{
14717 match(Set cr (CmpF src1 src2));
14718
14719 ins_cost(3 * INSN_COST);
14720 format %{ "fcmps $src1, $src2" %}
14721
14722 ins_encode %{
14723 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14724 %}
14725
14726 ins_pipe(pipe_class_compare);
14727 %}
14728
14729 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14730 %{
14731 match(Set cr (CmpF src1 src2));
14732
14733 ins_cost(3 * INSN_COST);
14734 format %{ "fcmps $src1, 0.0" %}
14735
14736 ins_encode %{
14737 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14738 %}
14739
14740 ins_pipe(pipe_class_compare);
14741 %}
14742 // FROM HERE
14743
14744 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14745 %{
14746 match(Set cr (CmpD src1 src2));
14747
14748 ins_cost(3 * INSN_COST);
14749 format %{ "fcmpd $src1, $src2" %}
14750
14751 ins_encode %{
14752 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14753 %}
14754
14755 ins_pipe(pipe_class_compare);
14756 %}
14757
14758 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14759 %{
14760 match(Set cr (CmpD src1 src2));
14761
14762 ins_cost(3 * INSN_COST);
14763 format %{ "fcmpd $src1, 0.0" %}
14764
14765 ins_encode %{
14766 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14767 %}
14768
14769 ins_pipe(pipe_class_compare);
14770 %}
14771
14772 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14773 %{
14774 match(Set dst (CmpF3 src1 src2));
14775 effect(KILL cr);
14776
14777 ins_cost(5 * INSN_COST);
14778 format %{ "fcmps $src1, $src2\n\t"
14779 "csinvw($dst, zr, zr, eq\n\t"
14780 "csnegw($dst, $dst, $dst, lt)"
14781 %}
14782
14783 ins_encode %{
14784 Label done;
14785 FloatRegister s1 = as_FloatRegister($src1$$reg);
14786 FloatRegister s2 = as_FloatRegister($src2$$reg);
14787 Register d = as_Register($dst$$reg);
14788 __ fcmps(s1, s2);
14789 // installs 0 if EQ else -1
14790 __ csinvw(d, zr, zr, Assembler::EQ);
14791 // keeps -1 if less or unordered else installs 1
14792 __ csnegw(d, d, d, Assembler::LT);
14793 __ bind(done);
14794 %}
14795
14796 ins_pipe(pipe_class_default);
14797
14798 %}
14799
14800 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14801 %{
14802 match(Set dst (CmpD3 src1 src2));
14803 effect(KILL cr);
14804
14805 ins_cost(5 * INSN_COST);
14806 format %{ "fcmpd $src1, $src2\n\t"
14807 "csinvw($dst, zr, zr, eq\n\t"
14808 "csnegw($dst, $dst, $dst, lt)"
14809 %}
14810
14811 ins_encode %{
14812 Label done;
14813 FloatRegister s1 = as_FloatRegister($src1$$reg);
14814 FloatRegister s2 = as_FloatRegister($src2$$reg);
14815 Register d = as_Register($dst$$reg);
14816 __ fcmpd(s1, s2);
14817 // installs 0 if EQ else -1
14818 __ csinvw(d, zr, zr, Assembler::EQ);
14819 // keeps -1 if less or unordered else installs 1
14820 __ csnegw(d, d, d, Assembler::LT);
14821 __ bind(done);
14822 %}
14823 ins_pipe(pipe_class_default);
14824
14825 %}
14826
14827 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14828 %{
14829 match(Set dst (CmpF3 src1 zero));
14830 effect(KILL cr);
14831
14832 ins_cost(5 * INSN_COST);
14833 format %{ "fcmps $src1, 0.0\n\t"
14834 "csinvw($dst, zr, zr, eq\n\t"
14835 "csnegw($dst, $dst, $dst, lt)"
14836 %}
14837
14838 ins_encode %{
14839 Label done;
14840 FloatRegister s1 = as_FloatRegister($src1$$reg);
14841 Register d = as_Register($dst$$reg);
14842 __ fcmps(s1, 0.0);
14843 // installs 0 if EQ else -1
14844 __ csinvw(d, zr, zr, Assembler::EQ);
14845 // keeps -1 if less or unordered else installs 1
14846 __ csnegw(d, d, d, Assembler::LT);
14847 __ bind(done);
14848 %}
14849
14850 ins_pipe(pipe_class_default);
14851
14852 %}
14853
14854 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14855 %{
14856 match(Set dst (CmpD3 src1 zero));
14857 effect(KILL cr);
14858
14859 ins_cost(5 * INSN_COST);
14860 format %{ "fcmpd $src1, 0.0\n\t"
14861 "csinvw($dst, zr, zr, eq\n\t"
14862 "csnegw($dst, $dst, $dst, lt)"
14863 %}
14864
14865 ins_encode %{
14866 Label done;
14867 FloatRegister s1 = as_FloatRegister($src1$$reg);
14868 Register d = as_Register($dst$$reg);
14869 __ fcmpd(s1, 0.0);
14870 // installs 0 if EQ else -1
14871 __ csinvw(d, zr, zr, Assembler::EQ);
14872 // keeps -1 if less or unordered else installs 1
14873 __ csnegw(d, d, d, Assembler::LT);
14874 __ bind(done);
14875 %}
14876 ins_pipe(pipe_class_default);
14877
14878 %}
14879
14880 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14881 %{
14882 match(Set dst (CmpLTMask p q));
14883 effect(KILL cr);
14884
14885 ins_cost(3 * INSN_COST);
14886
14887 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14888 "csetw $dst, lt\n\t"
14889 "subw $dst, zr, $dst"
14890 %}
14891
14892 ins_encode %{
14893 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14894 __ csetw(as_Register($dst$$reg), Assembler::LT);
14895 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14896 %}
14897
14898 ins_pipe(ialu_reg_reg);
14899 %}
14900
14901 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14902 %{
14903 match(Set dst (CmpLTMask src zero));
14904 effect(KILL cr);
14905
14906 ins_cost(INSN_COST);
14907
14908 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14909
14910 ins_encode %{
14911 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14912 %}
14913
14914 ins_pipe(ialu_reg_shift);
14915 %}
14916
14917 // ============================================================================
14918 // Max and Min
14919
14920 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14921
14922 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14923 %{
14924 effect(DEF cr, USE src);
14925 ins_cost(INSN_COST);
14926 format %{ "cmpw $src, 0" %}
14927
14928 ins_encode %{
14929 __ cmpw($src$$Register, 0);
14930 %}
14931 ins_pipe(icmp_reg_imm);
14932 %}
14933
14934 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14935 %{
14936 match(Set dst (MinI src1 src2));
14937 ins_cost(INSN_COST * 3);
14938
14939 expand %{
14940 rFlagsReg cr;
14941 compI_reg_reg(cr, src1, src2);
14942 cmovI_reg_reg_lt(dst, src1, src2, cr);
14943 %}
14944 %}
14945
14946 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14947 %{
14948 match(Set dst (MaxI src1 src2));
14949 ins_cost(INSN_COST * 3);
14950
14951 expand %{
14952 rFlagsReg cr;
14953 compI_reg_reg(cr, src1, src2);
14954 cmovI_reg_reg_gt(dst, src1, src2, cr);
14955 %}
14956 %}
14957
14958
14959 // ============================================================================
14960 // Branch Instructions
14961
14962 // Direct Branch.
14963 instruct branch(label lbl)
14964 %{
14965 match(Goto);
14966
14967 effect(USE lbl);
14968
14969 ins_cost(BRANCH_COST);
14970 format %{ "b $lbl" %}
14971
14972 ins_encode(aarch64_enc_b(lbl));
14973
14974 ins_pipe(pipe_branch);
14975 %}
14976
14977 // Conditional Near Branch
14978 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14979 %{
14980 // Same match rule as `branchConFar'.
14981 match(If cmp cr);
14982
14983 effect(USE lbl);
14984
14985 ins_cost(BRANCH_COST);
14986 // If set to 1 this indicates that the current instruction is a
14987 // short variant of a long branch. This avoids using this
14988 // instruction in first-pass matching. It will then only be used in
14989 // the `Shorten_branches' pass.
14990 // ins_short_branch(1);
14991 format %{ "b$cmp $lbl" %}
14992
14993 ins_encode(aarch64_enc_br_con(cmp, lbl));
14994
14995 ins_pipe(pipe_branch_cond);
14996 %}
14997
14998 // Conditional Near Branch Unsigned
14999 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15000 %{
15001 // Same match rule as `branchConFar'.
15002 match(If cmp cr);
15003
15004 effect(USE lbl);
15005
15006 ins_cost(BRANCH_COST);
15007 // If set to 1 this indicates that the current instruction is a
15008 // short variant of a long branch. This avoids using this
15009 // instruction in first-pass matching. It will then only be used in
15010 // the `Shorten_branches' pass.
15011 // ins_short_branch(1);
15012 format %{ "b$cmp $lbl\t# unsigned" %}
15013
15014 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15015
15016 ins_pipe(pipe_branch_cond);
15017 %}
15018
15019 // Make use of CBZ and CBNZ. These instructions, as well as being
15020 // shorter than (cmp; branch), have the additional benefit of not
15021 // killing the flags.
15022
15023 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15024 match(If cmp (CmpI op1 op2));
15025 effect(USE labl);
15026
15027 ins_cost(BRANCH_COST);
15028 format %{ "cbw$cmp $op1, $labl" %}
15029 ins_encode %{
15030 Label* L = $labl$$label;
15031 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15032 if (cond == Assembler::EQ)
15033 __ cbzw($op1$$Register, *L);
15034 else
15035 __ cbnzw($op1$$Register, *L);
15036 %}
15037 ins_pipe(pipe_cmp_branch);
15038 %}
15039
15040 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15041 match(If cmp (CmpL op1 op2));
15042 effect(USE labl);
15043
15044 ins_cost(BRANCH_COST);
15045 format %{ "cb$cmp $op1, $labl" %}
15046 ins_encode %{
15047 Label* L = $labl$$label;
15048 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15049 if (cond == Assembler::EQ)
15050 __ cbz($op1$$Register, *L);
15051 else
15052 __ cbnz($op1$$Register, *L);
15053 %}
15054 ins_pipe(pipe_cmp_branch);
15055 %}
15056
15057 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15058 match(If cmp (CmpP op1 op2));
15059 effect(USE labl);
15060
15061 ins_cost(BRANCH_COST);
15062 format %{ "cb$cmp $op1, $labl" %}
15063 ins_encode %{
15064 Label* L = $labl$$label;
15065 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15066 if (cond == Assembler::EQ)
15067 __ cbz($op1$$Register, *L);
15068 else
15069 __ cbnz($op1$$Register, *L);
15070 %}
15071 ins_pipe(pipe_cmp_branch);
15072 %}
15073
15074 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15075 match(If cmp (CmpN op1 op2));
15076 effect(USE labl);
15077
15078 ins_cost(BRANCH_COST);
15079 format %{ "cbw$cmp $op1, $labl" %}
15080 ins_encode %{
15081 Label* L = $labl$$label;
15082 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15083 if (cond == Assembler::EQ)
15084 __ cbzw($op1$$Register, *L);
15085 else
15086 __ cbnzw($op1$$Register, *L);
15087 %}
15088 ins_pipe(pipe_cmp_branch);
15089 %}
15090
15091 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15092 match(If cmp (CmpP (DecodeN oop) zero));
15093 effect(USE labl);
15094
15095 ins_cost(BRANCH_COST);
15096 format %{ "cb$cmp $oop, $labl" %}
15097 ins_encode %{
15098 Label* L = $labl$$label;
15099 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15100 if (cond == Assembler::EQ)
15101 __ cbzw($oop$$Register, *L);
15102 else
15103 __ cbnzw($oop$$Register, *L);
15104 %}
15105 ins_pipe(pipe_cmp_branch);
15106 %}
15107
15108 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15109 match(If cmp (CmpU op1 op2));
15110 effect(USE labl);
15111
15112 ins_cost(BRANCH_COST);
15113 format %{ "cbw$cmp $op1, $labl" %}
15114 ins_encode %{
15115 Label* L = $labl$$label;
15116 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15117 if (cond == Assembler::EQ || cond == Assembler::LS) {
15118 __ cbzw($op1$$Register, *L);
15119 } else {
15120 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15121 __ cbnzw($op1$$Register, *L);
15122 }
15123 %}
15124 ins_pipe(pipe_cmp_branch);
15125 %}
15126
15127 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15128 match(If cmp (CmpUL op1 op2));
15129 effect(USE labl);
15130
15131 ins_cost(BRANCH_COST);
15132 format %{ "cb$cmp $op1, $labl" %}
15133 ins_encode %{
15134 Label* L = $labl$$label;
15135 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15136 if (cond == Assembler::EQ || cond == Assembler::LS) {
15137 __ cbz($op1$$Register, *L);
15138 } else {
15139 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15140 __ cbnz($op1$$Register, *L);
15141 }
15142 %}
15143 ins_pipe(pipe_cmp_branch);
15144 %}
15145
15146 // Test bit and Branch
15147
15148 // Patterns for short (< 32KiB) variants
15149 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15150 match(If cmp (CmpL op1 op2));
15151 effect(USE labl);
15152
15153 ins_cost(BRANCH_COST);
15154 format %{ "cb$cmp $op1, $labl # long" %}
15155 ins_encode %{
15156 Label* L = $labl$$label;
15157 Assembler::Condition cond =
15158 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15159 __ tbr(cond, $op1$$Register, 63, *L);
15160 %}
15161 ins_pipe(pipe_cmp_branch);
15162 ins_short_branch(1);
15163 %}
15164
15165 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15166 match(If cmp (CmpI op1 op2));
15167 effect(USE labl);
15168
15169 ins_cost(BRANCH_COST);
15170 format %{ "cb$cmp $op1, $labl # int" %}
15171 ins_encode %{
15172 Label* L = $labl$$label;
15173 Assembler::Condition cond =
15174 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15175 __ tbr(cond, $op1$$Register, 31, *L);
15176 %}
15177 ins_pipe(pipe_cmp_branch);
15178 ins_short_branch(1);
15179 %}
15180
15181 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15182 match(If cmp (CmpL (AndL op1 op2) op3));
15183 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15184 effect(USE labl);
15185
15186 ins_cost(BRANCH_COST);
15187 format %{ "tb$cmp $op1, $op2, $labl" %}
15188 ins_encode %{
15189 Label* L = $labl$$label;
15190 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15191 int bit = exact_log2_long($op2$$constant);
15192 __ tbr(cond, $op1$$Register, bit, *L);
15193 %}
15194 ins_pipe(pipe_cmp_branch);
15195 ins_short_branch(1);
15196 %}
15197
15198 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15199 match(If cmp (CmpI (AndI op1 op2) op3));
15200 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15201 effect(USE labl);
15202
15203 ins_cost(BRANCH_COST);
15204 format %{ "tb$cmp $op1, $op2, $labl" %}
15205 ins_encode %{
15206 Label* L = $labl$$label;
15207 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15208 int bit = exact_log2((juint)$op2$$constant);
15209 __ tbr(cond, $op1$$Register, bit, *L);
15210 %}
15211 ins_pipe(pipe_cmp_branch);
15212 ins_short_branch(1);
15213 %}
15214
15215 // And far variants
15216 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15217 match(If cmp (CmpL op1 op2));
15218 effect(USE labl);
15219
15220 ins_cost(BRANCH_COST);
15221 format %{ "cb$cmp $op1, $labl # long" %}
15222 ins_encode %{
15223 Label* L = $labl$$label;
15224 Assembler::Condition cond =
15225 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15226 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15227 %}
15228 ins_pipe(pipe_cmp_branch);
15229 %}
15230
15231 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15232 match(If cmp (CmpI op1 op2));
15233 effect(USE labl);
15234
15235 ins_cost(BRANCH_COST);
15236 format %{ "cb$cmp $op1, $labl # int" %}
15237 ins_encode %{
15238 Label* L = $labl$$label;
15239 Assembler::Condition cond =
15240 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15241 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15242 %}
15243 ins_pipe(pipe_cmp_branch);
15244 %}
15245
15246 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15247 match(If cmp (CmpL (AndL op1 op2) op3));
15248 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15249 effect(USE labl);
15250
15251 ins_cost(BRANCH_COST);
15252 format %{ "tb$cmp $op1, $op2, $labl" %}
15253 ins_encode %{
15254 Label* L = $labl$$label;
15255 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15256 int bit = exact_log2_long($op2$$constant);
15257 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15258 %}
15259 ins_pipe(pipe_cmp_branch);
15260 %}
15261
15262 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15263 match(If cmp (CmpI (AndI op1 op2) op3));
15264 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15265 effect(USE labl);
15266
15267 ins_cost(BRANCH_COST);
15268 format %{ "tb$cmp $op1, $op2, $labl" %}
15269 ins_encode %{
15270 Label* L = $labl$$label;
15271 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15272 int bit = exact_log2((juint)$op2$$constant);
15273 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15274 %}
15275 ins_pipe(pipe_cmp_branch);
15276 %}
15277
15278 // Test bits
15279
15280 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15281 match(Set cr (CmpL (AndL op1 op2) op3));
15282 predicate(Assembler::operand_valid_for_logical_immediate
15283 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15284
15285 ins_cost(INSN_COST);
15286 format %{ "tst $op1, $op2 # long" %}
15287 ins_encode %{
15288 __ tst($op1$$Register, $op2$$constant);
15289 %}
15290 ins_pipe(ialu_reg_reg);
15291 %}
15292
15293 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15294 match(Set cr (CmpI (AndI op1 op2) op3));
15295 predicate(Assembler::operand_valid_for_logical_immediate
15296 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15297
15298 ins_cost(INSN_COST);
15299 format %{ "tst $op1, $op2 # int" %}
15300 ins_encode %{
15301 __ tstw($op1$$Register, $op2$$constant);
15302 %}
15303 ins_pipe(ialu_reg_reg);
15304 %}
15305
15306 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15307 match(Set cr (CmpL (AndL op1 op2) op3));
15308
15309 ins_cost(INSN_COST);
15310 format %{ "tst $op1, $op2 # long" %}
15311 ins_encode %{
15312 __ tst($op1$$Register, $op2$$Register);
15313 %}
15314 ins_pipe(ialu_reg_reg);
15315 %}
15316
15317 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15318 match(Set cr (CmpI (AndI op1 op2) op3));
15319
15320 ins_cost(INSN_COST);
15321 format %{ "tstw $op1, $op2 # int" %}
15322 ins_encode %{
15323 __ tstw($op1$$Register, $op2$$Register);
15324 %}
15325 ins_pipe(ialu_reg_reg);
15326 %}
15327
15328
15329 // Conditional Far Branch
15330 // Conditional Far Branch Unsigned
15331 // TODO: fixme
15332
15333 // counted loop end branch near
15334 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15335 %{
15336 match(CountedLoopEnd cmp cr);
15337
15338 effect(USE lbl);
15339
15340 ins_cost(BRANCH_COST);
15341 // short variant.
15342 // ins_short_branch(1);
15343 format %{ "b$cmp $lbl \t// counted loop end" %}
15344
15345 ins_encode(aarch64_enc_br_con(cmp, lbl));
15346
15347 ins_pipe(pipe_branch);
15348 %}
15349
15350 // counted loop end branch far
15351 // TODO: fixme
15352
15353 // ============================================================================
15354 // inlined locking and unlocking
15355
15356 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15357 %{
15358 match(Set cr (FastLock object box));
15359 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15360
15361 ins_cost(5 * INSN_COST);
15362 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15363
15364 ins_encode %{
15365 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15366 %}
15367
15368 ins_pipe(pipe_serial);
15369 %}
15370
15371 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15372 %{
15373 match(Set cr (FastUnlock object box));
15374 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15375
15376 ins_cost(5 * INSN_COST);
15377 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15378
15379 ins_encode %{
15380 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15381 %}
15382
15383 ins_pipe(pipe_serial);
15384 %}
15385
15386 // ============================================================================
15387 // Safepoint Instructions
15388
15389 // TODO
15390 // provide a near and far version of this code
15391
15392 instruct safePoint(rFlagsReg cr, iRegP poll)
15393 %{
15394 match(SafePoint poll);
15395 effect(KILL cr);
15396
15397 format %{
15398 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15399 %}
15400 ins_encode %{
15401 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15402 %}
15403 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15404 %}
15405
15406
15407 // ============================================================================
15408 // Procedure Call/Return Instructions
15409
15410 // Call Java Static Instruction
15411
15412 instruct CallStaticJavaDirect(method meth)
15413 %{
15414 match(CallStaticJava);
15415
15416 effect(USE meth);
15417
15418 ins_cost(CALL_COST);
15419
15420 format %{ "call,static $meth \t// ==> " %}
15421
15422 ins_encode(aarch64_enc_java_static_call(meth),
15423 aarch64_enc_call_epilog);
15424
15425 ins_pipe(pipe_class_call);
15426 %}
15427
15428 // TO HERE
15429
15430 // Call Java Dynamic Instruction
15431 instruct CallDynamicJavaDirect(method meth)
15432 %{
15433 match(CallDynamicJava);
15434
15435 effect(USE meth);
15436
15437 ins_cost(CALL_COST);
15438
15439 format %{ "CALL,dynamic $meth \t// ==> " %}
15440
15441 ins_encode(aarch64_enc_java_dynamic_call(meth),
15442 aarch64_enc_call_epilog);
15443
15444 ins_pipe(pipe_class_call);
15445 %}
15446
15447 // Call Runtime Instruction
15448
15449 instruct CallRuntimeDirect(method meth)
15450 %{
15451 match(CallRuntime);
15452
15453 effect(USE meth);
15454
15455 ins_cost(CALL_COST);
15456
15457 format %{ "CALL, runtime $meth" %}
15458
15459 ins_encode( aarch64_enc_java_to_runtime(meth) );
15460
15461 ins_pipe(pipe_class_call);
15462 %}
15463
15464 // Call Runtime Instruction
15465
15466 instruct CallLeafDirect(method meth)
15467 %{
15468 match(CallLeaf);
15469
15470 effect(USE meth);
15471
15472 ins_cost(CALL_COST);
15473
15474 format %{ "CALL, runtime leaf $meth" %}
15475
15476 ins_encode( aarch64_enc_java_to_runtime(meth) );
15477
15478 ins_pipe(pipe_class_call);
15479 %}
15480
15481 // Call Runtime Instruction without safepoint and with vector arguments
15482 instruct CallLeafDirectVector(method meth)
15483 %{
15484 match(CallLeafVector);
15485
15486 effect(USE meth);
15487
15488 ins_cost(CALL_COST);
15489
15490 format %{ "CALL, runtime leaf vector $meth" %}
15491
15492 ins_encode(aarch64_enc_java_to_runtime(meth));
15493
15494 ins_pipe(pipe_class_call);
15495 %}
15496
15497 // Call Runtime Instruction
15498
15499 // entry point is null, target holds the address to call
15500 instruct CallLeafNoFPIndirect(iRegP target)
15501 %{
15502 predicate(n->as_Call()->entry_point() == nullptr);
15503
15504 match(CallLeafNoFP target);
15505
15506 ins_cost(CALL_COST);
15507
15508 format %{ "CALL, runtime leaf nofp indirect $target" %}
15509
15510 ins_encode %{
15511 __ blr($target$$Register);
15512 %}
15513
15514 ins_pipe(pipe_class_call);
15515 %}
15516
15517 instruct CallLeafNoFPDirect(method meth)
15518 %{
15519 predicate(n->as_Call()->entry_point() != nullptr);
15520
15521 match(CallLeafNoFP);
15522
15523 effect(USE meth);
15524
15525 ins_cost(CALL_COST);
15526
15527 format %{ "CALL, runtime leaf nofp $meth" %}
15528
15529 ins_encode( aarch64_enc_java_to_runtime(meth) );
15530
15531 ins_pipe(pipe_class_call);
15532 %}
15533
15534 // Tail Call; Jump from runtime stub to Java code.
15535 // Also known as an 'interprocedural jump'.
15536 // Target of jump will eventually return to caller.
15537 // TailJump below removes the return address.
15538 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15539 // emitted just above the TailCall which has reset rfp to the caller state.
15540 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15541 %{
15542 match(TailCall jump_target method_ptr);
15543
15544 ins_cost(CALL_COST);
15545
15546 format %{ "br $jump_target\t# $method_ptr holds method" %}
15547
15548 ins_encode(aarch64_enc_tail_call(jump_target));
15549
15550 ins_pipe(pipe_class_call);
15551 %}
15552
15553 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15554 %{
15555 match(TailJump jump_target ex_oop);
15556
15557 ins_cost(CALL_COST);
15558
15559 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15560
15561 ins_encode(aarch64_enc_tail_jmp(jump_target));
15562
15563 ins_pipe(pipe_class_call);
15564 %}
15565
15566 // Forward exception.
15567 instruct ForwardExceptionjmp()
15568 %{
15569 match(ForwardException);
15570 ins_cost(CALL_COST);
15571
15572 format %{ "b forward_exception_stub" %}
15573 ins_encode %{
15574 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15575 %}
15576 ins_pipe(pipe_class_call);
15577 %}
15578
15579 // Create exception oop: created by stack-crawling runtime code.
15580 // Created exception is now available to this handler, and is setup
15581 // just prior to jumping to this handler. No code emitted.
15582 // TODO check
15583 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15584 instruct CreateException(iRegP_R0 ex_oop)
15585 %{
15586 match(Set ex_oop (CreateEx));
15587
15588 format %{ " -- \t// exception oop; no code emitted" %}
15589
15590 size(0);
15591
15592 ins_encode( /*empty*/ );
15593
15594 ins_pipe(pipe_class_empty);
15595 %}
15596
15597 // Rethrow exception: The exception oop will come in the first
15598 // argument position. Then JUMP (not call) to the rethrow stub code.
15599 instruct RethrowException() %{
15600 match(Rethrow);
15601 ins_cost(CALL_COST);
15602
15603 format %{ "b rethrow_stub" %}
15604
15605 ins_encode( aarch64_enc_rethrow() );
15606
15607 ins_pipe(pipe_class_call);
15608 %}
15609
15610
15611 // Return Instruction
15612 // epilog node loads ret address into lr as part of frame pop
15613 instruct Ret()
15614 %{
15615 match(Return);
15616
15617 format %{ "ret\t// return register" %}
15618
15619 ins_encode( aarch64_enc_ret() );
15620
15621 ins_pipe(pipe_branch);
15622 %}
15623
15624 // Die now.
15625 instruct ShouldNotReachHere() %{
15626 match(Halt);
15627
15628 ins_cost(CALL_COST);
15629 format %{ "ShouldNotReachHere" %}
15630
15631 ins_encode %{
15632 if (is_reachable()) {
15633 const char* str = __ code_string(_halt_reason);
15634 __ stop(str);
15635 }
15636 %}
15637
15638 ins_pipe(pipe_class_default);
15639 %}
15640
15641 // ============================================================================
15642 // Partial Subtype Check
15643 //
15644 // superklass array for an instance of the superklass. Set a hidden
15645 // internal cache on a hit (cache is checked with exposed code in
15646 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15647 // encoding ALSO sets flags.
15648
15649 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15650 %{
15651 match(Set result (PartialSubtypeCheck sub super));
15652 predicate(!UseSecondarySupersTable);
15653 effect(KILL cr, KILL temp);
15654
15655 ins_cost(20 * INSN_COST); // slightly larger than the next version
15656 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15657
15658 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15659
15660 opcode(0x1); // Force zero of result reg on hit
15661
15662 ins_pipe(pipe_class_memory);
15663 %}
15664
15665 // Two versions of partialSubtypeCheck, both used when we need to
15666 // search for a super class in the secondary supers array. The first
15667 // is used when we don't know _a priori_ the class being searched
15668 // for. The second, far more common, is used when we do know: this is
15669 // used for instanceof, checkcast, and any case where C2 can determine
15670 // it by constant propagation.
15671
15672 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15673 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15674 rFlagsReg cr)
15675 %{
15676 match(Set result (PartialSubtypeCheck sub super));
15677 predicate(UseSecondarySupersTable);
15678 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15679
15680 ins_cost(10 * INSN_COST); // slightly larger than the next version
15681 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15682
15683 ins_encode %{
15684 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15685 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15686 $vtemp$$FloatRegister,
15687 $result$$Register, /*L_success*/nullptr);
15688 %}
15689
15690 ins_pipe(pipe_class_memory);
15691 %}
15692
15693 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15694 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15695 rFlagsReg cr)
15696 %{
15697 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15698 predicate(UseSecondarySupersTable);
15699 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15700
15701 ins_cost(5 * INSN_COST); // smaller than the next version
15702 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15703
15704 ins_encode %{
15705 bool success = false;
15706 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15707 if (InlineSecondarySupersTest) {
15708 success =
15709 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15710 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15711 $vtemp$$FloatRegister,
15712 $result$$Register,
15713 super_klass_slot);
15714 } else {
15715 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15716 success = (call != nullptr);
15717 }
15718 if (!success) {
15719 ciEnv::current()->record_failure("CodeCache is full");
15720 return;
15721 }
15722 %}
15723
15724 ins_pipe(pipe_class_memory);
15725 %}
15726
15727 // Intrisics for String.compareTo()
15728
15729 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15730 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15731 %{
15732 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15733 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15734 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15735
15736 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15737 ins_encode %{
15738 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15739 __ string_compare($str1$$Register, $str2$$Register,
15740 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15741 $tmp1$$Register, $tmp2$$Register,
15742 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15743 %}
15744 ins_pipe(pipe_class_memory);
15745 %}
15746
15747 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15748 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15749 %{
15750 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15751 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15752 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15753
15754 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15755 ins_encode %{
15756 __ string_compare($str1$$Register, $str2$$Register,
15757 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15758 $tmp1$$Register, $tmp2$$Register,
15759 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15760 %}
15761 ins_pipe(pipe_class_memory);
15762 %}
15763
15764 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15765 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15766 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15767 %{
15768 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15769 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15770 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15771 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15772
15773 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15774 ins_encode %{
15775 __ string_compare($str1$$Register, $str2$$Register,
15776 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15777 $tmp1$$Register, $tmp2$$Register,
15778 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15779 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15780 %}
15781 ins_pipe(pipe_class_memory);
15782 %}
15783
15784 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15785 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15786 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15787 %{
15788 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15789 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15790 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15791 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15792
15793 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15794 ins_encode %{
15795 __ string_compare($str1$$Register, $str2$$Register,
15796 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15797 $tmp1$$Register, $tmp2$$Register,
15798 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15799 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15800 %}
15801 ins_pipe(pipe_class_memory);
15802 %}
15803
15804 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15805 // these string_compare variants as NEON register type for convenience so that the prototype of
15806 // string_compare can be shared with all variants.
15807
15808 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15809 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15810 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15811 pRegGov_P1 pgtmp2, rFlagsReg cr)
15812 %{
15813 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15814 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15815 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15816 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15817
15818 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15819 ins_encode %{
15820 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15821 __ string_compare($str1$$Register, $str2$$Register,
15822 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15823 $tmp1$$Register, $tmp2$$Register,
15824 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15825 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15826 StrIntrinsicNode::LL);
15827 %}
15828 ins_pipe(pipe_class_memory);
15829 %}
15830
15831 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15832 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15833 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15834 pRegGov_P1 pgtmp2, rFlagsReg cr)
15835 %{
15836 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15837 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15838 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15839 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15840
15841 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15842 ins_encode %{
15843 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15844 __ string_compare($str1$$Register, $str2$$Register,
15845 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15846 $tmp1$$Register, $tmp2$$Register,
15847 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15848 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15849 StrIntrinsicNode::LU);
15850 %}
15851 ins_pipe(pipe_class_memory);
15852 %}
15853
15854 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15855 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15856 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15857 pRegGov_P1 pgtmp2, rFlagsReg cr)
15858 %{
15859 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15860 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15861 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15862 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15863
15864 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15865 ins_encode %{
15866 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15867 __ string_compare($str1$$Register, $str2$$Register,
15868 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15869 $tmp1$$Register, $tmp2$$Register,
15870 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15871 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15872 StrIntrinsicNode::UL);
15873 %}
15874 ins_pipe(pipe_class_memory);
15875 %}
15876
15877 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15878 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15879 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15880 pRegGov_P1 pgtmp2, rFlagsReg cr)
15881 %{
15882 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15883 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15884 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15885 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15886
15887 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15888 ins_encode %{
15889 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15890 __ string_compare($str1$$Register, $str2$$Register,
15891 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15892 $tmp1$$Register, $tmp2$$Register,
15893 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15894 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15895 StrIntrinsicNode::UU);
15896 %}
15897 ins_pipe(pipe_class_memory);
15898 %}
15899
15900 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15901 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15902 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15903 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15904 %{
15905 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15906 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15907 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15908 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15909 TEMP vtmp0, TEMP vtmp1, KILL cr);
15910 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15911 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15912
15913 ins_encode %{
15914 __ string_indexof($str1$$Register, $str2$$Register,
15915 $cnt1$$Register, $cnt2$$Register,
15916 $tmp1$$Register, $tmp2$$Register,
15917 $tmp3$$Register, $tmp4$$Register,
15918 $tmp5$$Register, $tmp6$$Register,
15919 -1, $result$$Register, StrIntrinsicNode::UU);
15920 %}
15921 ins_pipe(pipe_class_memory);
15922 %}
15923
15924 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15925 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15926 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15927 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15928 %{
15929 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15930 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15931 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15932 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15933 TEMP vtmp0, TEMP vtmp1, KILL cr);
15934 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15935 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15936
15937 ins_encode %{
15938 __ string_indexof($str1$$Register, $str2$$Register,
15939 $cnt1$$Register, $cnt2$$Register,
15940 $tmp1$$Register, $tmp2$$Register,
15941 $tmp3$$Register, $tmp4$$Register,
15942 $tmp5$$Register, $tmp6$$Register,
15943 -1, $result$$Register, StrIntrinsicNode::LL);
15944 %}
15945 ins_pipe(pipe_class_memory);
15946 %}
15947
15948 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15949 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15950 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15951 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15952 %{
15953 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15954 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15955 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15956 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15957 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15958 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15959 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15960
15961 ins_encode %{
15962 __ string_indexof($str1$$Register, $str2$$Register,
15963 $cnt1$$Register, $cnt2$$Register,
15964 $tmp1$$Register, $tmp2$$Register,
15965 $tmp3$$Register, $tmp4$$Register,
15966 $tmp5$$Register, $tmp6$$Register,
15967 -1, $result$$Register, StrIntrinsicNode::UL);
15968 %}
15969 ins_pipe(pipe_class_memory);
15970 %}
15971
15972 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15973 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15974 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15975 %{
15976 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15977 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15978 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15979 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15980 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15981 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15982
15983 ins_encode %{
15984 int icnt2 = (int)$int_cnt2$$constant;
15985 __ string_indexof($str1$$Register, $str2$$Register,
15986 $cnt1$$Register, zr,
15987 $tmp1$$Register, $tmp2$$Register,
15988 $tmp3$$Register, $tmp4$$Register, zr, zr,
15989 icnt2, $result$$Register, StrIntrinsicNode::UU);
15990 %}
15991 ins_pipe(pipe_class_memory);
15992 %}
15993
15994 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15995 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15996 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15997 %{
15998 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15999 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16000 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16001 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16002 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16003 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16004
16005 ins_encode %{
16006 int icnt2 = (int)$int_cnt2$$constant;
16007 __ string_indexof($str1$$Register, $str2$$Register,
16008 $cnt1$$Register, zr,
16009 $tmp1$$Register, $tmp2$$Register,
16010 $tmp3$$Register, $tmp4$$Register, zr, zr,
16011 icnt2, $result$$Register, StrIntrinsicNode::LL);
16012 %}
16013 ins_pipe(pipe_class_memory);
16014 %}
16015
16016 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16017 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16018 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16019 %{
16020 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16021 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16022 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16023 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16024 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16025 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16026
16027 ins_encode %{
16028 int icnt2 = (int)$int_cnt2$$constant;
16029 __ string_indexof($str1$$Register, $str2$$Register,
16030 $cnt1$$Register, zr,
16031 $tmp1$$Register, $tmp2$$Register,
16032 $tmp3$$Register, $tmp4$$Register, zr, zr,
16033 icnt2, $result$$Register, StrIntrinsicNode::UL);
16034 %}
16035 ins_pipe(pipe_class_memory);
16036 %}
16037
16038 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16039 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16040 iRegINoSp tmp3, rFlagsReg cr)
16041 %{
16042 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16043 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16044 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16045 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16046
16047 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16048
16049 ins_encode %{
16050 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16051 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16052 $tmp3$$Register);
16053 %}
16054 ins_pipe(pipe_class_memory);
16055 %}
16056
16057 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16058 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16059 iRegINoSp tmp3, rFlagsReg cr)
16060 %{
16061 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16062 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16063 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16064 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16065
16066 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16067
16068 ins_encode %{
16069 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16070 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16071 $tmp3$$Register);
16072 %}
16073 ins_pipe(pipe_class_memory);
16074 %}
16075
16076 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16077 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16078 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16079 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16080 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16081 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16082 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16083 ins_encode %{
16084 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16085 $result$$Register, $ztmp1$$FloatRegister,
16086 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16087 $ptmp$$PRegister, true /* isL */);
16088 %}
16089 ins_pipe(pipe_class_memory);
16090 %}
16091
16092 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16093 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16094 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16095 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16096 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16097 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16098 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16099 ins_encode %{
16100 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16101 $result$$Register, $ztmp1$$FloatRegister,
16102 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16103 $ptmp$$PRegister, false /* isL */);
16104 %}
16105 ins_pipe(pipe_class_memory);
16106 %}
16107
16108 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16109 iRegI_R0 result, rFlagsReg cr)
16110 %{
16111 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16112 match(Set result (StrEquals (Binary str1 str2) cnt));
16113 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16114
16115 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16116 ins_encode %{
16117 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16118 __ string_equals($str1$$Register, $str2$$Register,
16119 $result$$Register, $cnt$$Register);
16120 %}
16121 ins_pipe(pipe_class_memory);
16122 %}
16123
16124 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16125 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16126 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16127 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16128 iRegP_R10 tmp, rFlagsReg cr)
16129 %{
16130 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16131 match(Set result (AryEq ary1 ary2));
16132 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16133 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16134 TEMP vtmp6, TEMP vtmp7, KILL cr);
16135
16136 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16137 ins_encode %{
16138 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16139 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16140 $result$$Register, $tmp$$Register, 1);
16141 if (tpc == nullptr) {
16142 ciEnv::current()->record_failure("CodeCache is full");
16143 return;
16144 }
16145 %}
16146 ins_pipe(pipe_class_memory);
16147 %}
16148
16149 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16150 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16151 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16152 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16153 iRegP_R10 tmp, rFlagsReg cr)
16154 %{
16155 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16156 match(Set result (AryEq ary1 ary2));
16157 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16158 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16159 TEMP vtmp6, TEMP vtmp7, KILL cr);
16160
16161 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16162 ins_encode %{
16163 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16164 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16165 $result$$Register, $tmp$$Register, 2);
16166 if (tpc == nullptr) {
16167 ciEnv::current()->record_failure("CodeCache is full");
16168 return;
16169 }
16170 %}
16171 ins_pipe(pipe_class_memory);
16172 %}
16173
16174 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16175 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16176 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16177 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16178 %{
16179 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16180 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16181 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16182
16183 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16184 ins_encode %{
16185 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16186 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16187 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16188 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16189 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16190 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16191 (BasicType)$basic_type$$constant);
16192 if (tpc == nullptr) {
16193 ciEnv::current()->record_failure("CodeCache is full");
16194 return;
16195 }
16196 %}
16197 ins_pipe(pipe_class_memory);
16198 %}
16199
16200 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16201 %{
16202 match(Set result (CountPositives ary1 len));
16203 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16204 format %{ "count positives byte[] $ary1,$len -> $result" %}
16205 ins_encode %{
16206 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16207 if (tpc == nullptr) {
16208 ciEnv::current()->record_failure("CodeCache is full");
16209 return;
16210 }
16211 %}
16212 ins_pipe( pipe_slow );
16213 %}
16214
16215 // fast char[] to byte[] compression
16216 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16217 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16218 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16219 iRegI_R0 result, rFlagsReg cr)
16220 %{
16221 match(Set result (StrCompressedCopy src (Binary dst len)));
16222 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16223 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16224
16225 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16226 ins_encode %{
16227 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16228 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16229 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16230 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16231 %}
16232 ins_pipe(pipe_slow);
16233 %}
16234
16235 // fast byte[] to char[] inflation
16236 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16237 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16238 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16239 %{
16240 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16241 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16242 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16243 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16244
16245 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16246 ins_encode %{
16247 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16248 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16249 $vtmp2$$FloatRegister, $tmp$$Register);
16250 if (tpc == nullptr) {
16251 ciEnv::current()->record_failure("CodeCache is full");
16252 return;
16253 }
16254 %}
16255 ins_pipe(pipe_class_memory);
16256 %}
16257
16258 // encode char[] to byte[] in ISO_8859_1
16259 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16260 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16261 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16262 iRegI_R0 result, rFlagsReg cr)
16263 %{
16264 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16265 match(Set result (EncodeISOArray src (Binary dst len)));
16266 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16267 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16268
16269 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16270 ins_encode %{
16271 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16272 $result$$Register, false,
16273 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16274 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16275 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16276 %}
16277 ins_pipe(pipe_class_memory);
16278 %}
16279
16280 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16281 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16282 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16283 iRegI_R0 result, rFlagsReg cr)
16284 %{
16285 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16286 match(Set result (EncodeISOArray src (Binary dst len)));
16287 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16288 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16289
16290 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16291 ins_encode %{
16292 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16293 $result$$Register, true,
16294 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16295 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16296 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16297 %}
16298 ins_pipe(pipe_class_memory);
16299 %}
16300
16301 //----------------------------- CompressBits/ExpandBits ------------------------
16302
16303 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16304 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16305 match(Set dst (CompressBits src mask));
16306 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16307 format %{ "mov $tsrc, $src\n\t"
16308 "mov $tmask, $mask\n\t"
16309 "bext $tdst, $tsrc, $tmask\n\t"
16310 "mov $dst, $tdst"
16311 %}
16312 ins_encode %{
16313 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16314 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16315 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16316 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16317 %}
16318 ins_pipe(pipe_slow);
16319 %}
16320
16321 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16322 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16323 match(Set dst (CompressBits (LoadI mem) mask));
16324 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16325 format %{ "ldrs $tsrc, $mem\n\t"
16326 "ldrs $tmask, $mask\n\t"
16327 "bext $tdst, $tsrc, $tmask\n\t"
16328 "mov $dst, $tdst"
16329 %}
16330 ins_encode %{
16331 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16332 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16333 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16334 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16335 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16336 %}
16337 ins_pipe(pipe_slow);
16338 %}
16339
16340 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16341 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16342 match(Set dst (CompressBits src mask));
16343 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16344 format %{ "mov $tsrc, $src\n\t"
16345 "mov $tmask, $mask\n\t"
16346 "bext $tdst, $tsrc, $tmask\n\t"
16347 "mov $dst, $tdst"
16348 %}
16349 ins_encode %{
16350 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16351 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16352 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16353 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16354 %}
16355 ins_pipe(pipe_slow);
16356 %}
16357
16358 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16359 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16360 match(Set dst (CompressBits (LoadL mem) mask));
16361 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16362 format %{ "ldrd $tsrc, $mem\n\t"
16363 "ldrd $tmask, $mask\n\t"
16364 "bext $tdst, $tsrc, $tmask\n\t"
16365 "mov $dst, $tdst"
16366 %}
16367 ins_encode %{
16368 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16369 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16370 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16371 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16372 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16373 %}
16374 ins_pipe(pipe_slow);
16375 %}
16376
16377 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16378 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16379 match(Set dst (ExpandBits src mask));
16380 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16381 format %{ "mov $tsrc, $src\n\t"
16382 "mov $tmask, $mask\n\t"
16383 "bdep $tdst, $tsrc, $tmask\n\t"
16384 "mov $dst, $tdst"
16385 %}
16386 ins_encode %{
16387 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16388 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16389 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16390 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16391 %}
16392 ins_pipe(pipe_slow);
16393 %}
16394
16395 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16396 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16397 match(Set dst (ExpandBits (LoadI mem) mask));
16398 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16399 format %{ "ldrs $tsrc, $mem\n\t"
16400 "ldrs $tmask, $mask\n\t"
16401 "bdep $tdst, $tsrc, $tmask\n\t"
16402 "mov $dst, $tdst"
16403 %}
16404 ins_encode %{
16405 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16406 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16407 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16408 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16409 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16410 %}
16411 ins_pipe(pipe_slow);
16412 %}
16413
16414 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16415 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16416 match(Set dst (ExpandBits src mask));
16417 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16418 format %{ "mov $tsrc, $src\n\t"
16419 "mov $tmask, $mask\n\t"
16420 "bdep $tdst, $tsrc, $tmask\n\t"
16421 "mov $dst, $tdst"
16422 %}
16423 ins_encode %{
16424 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16425 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16426 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16427 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16428 %}
16429 ins_pipe(pipe_slow);
16430 %}
16431
16432
16433 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16434 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16435 match(Set dst (ExpandBits (LoadL mem) mask));
16436 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16437 format %{ "ldrd $tsrc, $mem\n\t"
16438 "ldrd $tmask, $mask\n\t"
16439 "bdep $tdst, $tsrc, $tmask\n\t"
16440 "mov $dst, $tdst"
16441 %}
16442 ins_encode %{
16443 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16444 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16445 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16446 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16447 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16448 %}
16449 ins_pipe(pipe_slow);
16450 %}
16451
16452 //----------------------------- Reinterpret ----------------------------------
16453 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16454 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16455 match(Set dst (ReinterpretHF2S src));
16456 format %{ "reinterpretHF2S $dst, $src" %}
16457 ins_encode %{
16458 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16459 %}
16460 ins_pipe(pipe_slow);
16461 %}
16462
16463 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16464 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16465 match(Set dst (ReinterpretS2HF src));
16466 format %{ "reinterpretS2HF $dst, $src" %}
16467 ins_encode %{
16468 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16469 %}
16470 ins_pipe(pipe_slow);
16471 %}
16472
16473 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16474 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16475 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16476 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16477 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16478 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16479 // can be omitted in this pattern, resulting in -
16480 // fcvt $dst, $src // Convert float to half-precision float
16481 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16482 %{
16483 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16484 format %{ "convF2HFAndS2HF $dst, $src" %}
16485 ins_encode %{
16486 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16487 %}
16488 ins_pipe(pipe_slow);
16489 %}
16490
16491 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16492 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16493 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16494 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16495 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16496 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16497 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16498 // resulting in -
16499 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16500 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16501 %{
16502 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16503 format %{ "convHF2SAndHF2F $dst, $src" %}
16504 ins_encode %{
16505 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16506 %}
16507 ins_pipe(pipe_slow);
16508 %}
16509
16510 // ============================================================================
16511 // This name is KNOWN by the ADLC and cannot be changed.
16512 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16513 // for this guy.
16514 instruct tlsLoadP(thread_RegP dst)
16515 %{
16516 match(Set dst (ThreadLocal));
16517
16518 ins_cost(0);
16519
16520 format %{ " -- \t// $dst=Thread::current(), empty" %}
16521
16522 size(0);
16523
16524 ins_encode( /*empty*/ );
16525
16526 ins_pipe(pipe_class_empty);
16527 %}
16528
16529 //----------PEEPHOLE RULES-----------------------------------------------------
16530 // These must follow all instruction definitions as they use the names
16531 // defined in the instructions definitions.
16532 //
16533 // peepmatch ( root_instr_name [preceding_instruction]* );
16534 //
16535 // peepconstraint %{
16536 // (instruction_number.operand_name relational_op instruction_number.operand_name
16537 // [, ...] );
16538 // // instruction numbers are zero-based using left to right order in peepmatch
16539 //
16540 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16541 // // provide an instruction_number.operand_name for each operand that appears
16542 // // in the replacement instruction's match rule
16543 //
16544 // ---------VM FLAGS---------------------------------------------------------
16545 //
16546 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16547 //
16548 // Each peephole rule is given an identifying number starting with zero and
16549 // increasing by one in the order seen by the parser. An individual peephole
16550 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16551 // on the command-line.
16552 //
16553 // ---------CURRENT LIMITATIONS----------------------------------------------
16554 //
16555 // Only match adjacent instructions in same basic block
16556 // Only equality constraints
16557 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16558 // Only one replacement instruction
16559 //
16560 // ---------EXAMPLE----------------------------------------------------------
16561 //
16562 // // pertinent parts of existing instructions in architecture description
16563 // instruct movI(iRegINoSp dst, iRegI src)
16564 // %{
16565 // match(Set dst (CopyI src));
16566 // %}
16567 //
16568 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16569 // %{
16570 // match(Set dst (AddI dst src));
16571 // effect(KILL cr);
16572 // %}
16573 //
16574 // // Change (inc mov) to lea
16575 // peephole %{
16576 // // increment preceded by register-register move
16577 // peepmatch ( incI_iReg movI );
16578 // // require that the destination register of the increment
16579 // // match the destination register of the move
16580 // peepconstraint ( 0.dst == 1.dst );
16581 // // construct a replacement instruction that sets
16582 // // the destination to ( move's source register + one )
16583 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16584 // %}
16585 //
16586
16587 // Implementation no longer uses movX instructions since
16588 // machine-independent system no longer uses CopyX nodes.
16589 //
16590 // peephole
16591 // %{
16592 // peepmatch (incI_iReg movI);
16593 // peepconstraint (0.dst == 1.dst);
16594 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16595 // %}
16596
16597 // peephole
16598 // %{
16599 // peepmatch (decI_iReg movI);
16600 // peepconstraint (0.dst == 1.dst);
16601 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16602 // %}
16603
16604 // peephole
16605 // %{
16606 // peepmatch (addI_iReg_imm movI);
16607 // peepconstraint (0.dst == 1.dst);
16608 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16609 // %}
16610
16611 // peephole
16612 // %{
16613 // peepmatch (incL_iReg movL);
16614 // peepconstraint (0.dst == 1.dst);
16615 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16616 // %}
16617
16618 // peephole
16619 // %{
16620 // peepmatch (decL_iReg movL);
16621 // peepconstraint (0.dst == 1.dst);
16622 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16623 // %}
16624
16625 // peephole
16626 // %{
16627 // peepmatch (addL_iReg_imm movL);
16628 // peepconstraint (0.dst == 1.dst);
16629 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16630 // %}
16631
16632 // peephole
16633 // %{
16634 // peepmatch (addP_iReg_imm movP);
16635 // peepconstraint (0.dst == 1.dst);
16636 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16637 // %}
16638
16639 // // Change load of spilled value to only a spill
16640 // instruct storeI(memory mem, iRegI src)
16641 // %{
16642 // match(Set mem (StoreI mem src));
16643 // %}
16644 //
16645 // instruct loadI(iRegINoSp dst, memory mem)
16646 // %{
16647 // match(Set dst (LoadI mem));
16648 // %}
16649 //
16650
16651 //----------SMARTSPILL RULES---------------------------------------------------
16652 // These must follow all instruction definitions as they use the names
16653 // defined in the instructions definitions.
16654
16655 // Local Variables:
16656 // mode: c++
16657 // End: