1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1713
1714 //=============================================================================
1715
1716 #ifndef PRODUCT
1717 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1718 st->print("nop \t# %d bytes pad for loops and calls", _count);
1719 }
1720 #endif
1721
1722 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1723 for (int i = 0; i < _count; i++) {
1724 __ nop();
1725 }
1726 }
1727
1728 uint MachNopNode::size(PhaseRegAlloc*) const {
1729 return _count * NativeInstruction::instruction_size;
1730 }
1731
1732 //=============================================================================
1733 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1734
1735 int ConstantTable::calculate_table_base_offset() const {
1736 return 0; // absolute addressing, no offset
1737 }
1738
1739 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1740 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1741 ShouldNotReachHere();
1742 }
1743
1744 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1745 // Empty encoding
1746 }
1747
1748 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1749 return 0;
1750 }
1751
1752 #ifndef PRODUCT
1753 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1754 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1755 }
1756 #endif
1757
1758 #ifndef PRODUCT
1759 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1760 Compile* C = ra_->C;
1761
1762 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1763
1764 if (C->output()->need_stack_bang(framesize))
1765 st->print("# stack bang size=%d\n\t", framesize);
1766
1767 if (VM_Version::use_rop_protection()) {
1768 st->print("ldr zr, [lr]\n\t");
1769 st->print("paciaz\n\t");
1770 }
1771 if (framesize < ((1 << 9) + 2 * wordSize)) {
1772 st->print("sub sp, sp, #%d\n\t", framesize);
1773 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1774 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1775 } else {
1776 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1777 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1778 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1779 st->print("sub sp, sp, rscratch1");
1780 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1886 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1887 st->print("add sp, sp, rscratch1\n\t");
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
1952 if (reg < slots_of_int_registers) {
1953 return rc_int;
1954 }
1955
1956 // we have 32 float register * 8 halves
1957 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1958 if (reg < slots_of_int_registers + slots_of_float_registers) {
1959 return rc_float;
1960 }
1961
1962 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1963 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1964 return rc_predicate;
1965 }
1966
1967 // Between predicate regs & stack is the flags.
1968 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1969
1970 return rc_stack;
1971 }
1972
1973 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1974 Compile* C = ra_->C;
1975
1976 // Get registers to move.
1977 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1978 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1979 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1980 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1981
1982 enum RC src_hi_rc = rc_class(src_hi);
1983 enum RC src_lo_rc = rc_class(src_lo);
1984 enum RC dst_hi_rc = rc_class(dst_hi);
1985 enum RC dst_lo_rc = rc_class(dst_lo);
1986
1987 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1988
1989 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1990 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1991 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1992 "expected aligned-adjacent pairs");
1993 }
1994
1995 if (src_lo == dst_lo && src_hi == dst_hi) {
1996 return 0; // Self copy, no move.
1997 }
1998
1999 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2000 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2001 int src_offset = ra_->reg2offset(src_lo);
2002 int dst_offset = ra_->reg2offset(dst_lo);
2003
2004 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2005 uint ireg = ideal_reg();
2006 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2007 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2008 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2009 if (ireg == Op_VecA && masm) {
2010 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2011 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2012 // stack->stack
2013 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2016 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2017 sve_vector_reg_size_in_bytes);
2018 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2019 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2020 sve_vector_reg_size_in_bytes);
2021 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2022 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2023 as_FloatRegister(Matcher::_regEncode[src_lo]),
2024 as_FloatRegister(Matcher::_regEncode[src_lo]));
2025 } else {
2026 ShouldNotReachHere();
2027 }
2028 } else if (masm) {
2029 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2030 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2031 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2032 // stack->stack
2033 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2034 if (ireg == Op_VecD) {
2035 __ unspill(rscratch1, true, src_offset);
2036 __ spill(rscratch1, true, dst_offset);
2037 } else {
2038 __ spill_copy128(src_offset, dst_offset);
2039 }
2040 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2041 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 ireg == Op_VecD ? __ T8B : __ T16B,
2043 as_FloatRegister(Matcher::_regEncode[src_lo]));
2044 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2045 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2046 ireg == Op_VecD ? __ D : __ Q,
2047 ra_->reg2offset(dst_lo));
2048 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2049 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 ireg == Op_VecD ? __ D : __ Q,
2051 ra_->reg2offset(src_lo));
2052 } else {
2053 ShouldNotReachHere();
2054 }
2055 }
2056 } else if (masm) {
2057 switch (src_lo_rc) {
2058 case rc_int:
2059 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2060 if (is64) {
2061 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2062 as_Register(Matcher::_regEncode[src_lo]));
2063 } else {
2064 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2065 as_Register(Matcher::_regEncode[src_lo]));
2066 }
2067 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2068 if (is64) {
2069 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2070 as_Register(Matcher::_regEncode[src_lo]));
2071 } else {
2072 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 as_Register(Matcher::_regEncode[src_lo]));
2074 }
2075 } else { // gpr --> stack spill
2076 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2077 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2078 }
2079 break;
2080 case rc_float:
2081 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2082 if (is64) {
2083 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2084 as_FloatRegister(Matcher::_regEncode[src_lo]));
2085 } else {
2086 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2087 as_FloatRegister(Matcher::_regEncode[src_lo]));
2088 }
2089 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2090 if (is64) {
2091 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2092 as_FloatRegister(Matcher::_regEncode[src_lo]));
2093 } else {
2094 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 }
2097 } else { // fpr --> stack spill
2098 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2099 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2100 is64 ? __ D : __ S, dst_offset);
2101 }
2102 break;
2103 case rc_stack:
2104 if (dst_lo_rc == rc_int) { // stack --> gpr load
2105 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2106 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2107 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2108 is64 ? __ D : __ S, src_offset);
2109 } else if (dst_lo_rc == rc_predicate) {
2110 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2111 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2112 } else { // stack --> stack copy
2113 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2114 if (ideal_reg() == Op_RegVectMask) {
2115 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2116 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2117 } else {
2118 __ unspill(rscratch1, is64, src_offset);
2119 __ spill(rscratch1, is64, dst_offset);
2120 }
2121 }
2122 break;
2123 case rc_predicate:
2124 if (dst_lo_rc == rc_predicate) {
2125 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2126 } else if (dst_lo_rc == rc_stack) {
2127 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2128 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2129 } else {
2130 assert(false, "bad src and dst rc_class combination.");
2131 ShouldNotReachHere();
2132 }
2133 break;
2134 default:
2135 assert(false, "bad rc_class for spill");
2136 ShouldNotReachHere();
2137 }
2138 }
2139
2140 if (st) {
2141 st->print("spill ");
2142 if (src_lo_rc == rc_stack) {
2143 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2144 } else {
2145 st->print("%s -> ", Matcher::regName[src_lo]);
2146 }
2147 if (dst_lo_rc == rc_stack) {
2148 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2149 } else {
2150 st->print("%s", Matcher::regName[dst_lo]);
2151 }
2152 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2153 int vsize = 0;
2154 switch (ideal_reg()) {
2155 case Op_VecD:
2156 vsize = 64;
2157 break;
2158 case Op_VecX:
2159 vsize = 128;
2160 break;
2161 case Op_VecA:
2162 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2163 break;
2164 default:
2165 assert(false, "bad register type for spill");
2166 ShouldNotReachHere();
2167 }
2168 st->print("\t# vector spill size = %d", vsize);
2169 } else if (ideal_reg() == Op_RegVectMask) {
2170 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2171 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2172 st->print("\t# predicate spill size = %d", vsize);
2173 } else {
2174 st->print("\t# spill size = %d", is64 ? 64 : 32);
2175 }
2176 }
2177
2178 return 0;
2179
2180 }
2181
2182 #ifndef PRODUCT
2183 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2184 if (!ra_)
2185 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2186 else
2187 implementation(nullptr, ra_, false, st);
2188 }
2189 #endif
2190
2191 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2192 implementation(masm, ra_, false, nullptr);
2193 }
2194
2195 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2196 return MachNode::size(ra_);
2197 }
2198
2199 //=============================================================================
2200
2201 #ifndef PRODUCT
2202 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2203 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2204 int reg = ra_->get_reg_first(this);
2205 st->print("add %s, rsp, #%d]\t# box lock",
2206 Matcher::regName[reg], offset);
2207 }
2208 #endif
2209
2210 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2211 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2212 int reg = ra_->get_encode(this);
2213
2214 // This add will handle any 24-bit signed offset. 24 bits allows an
2215 // 8 megabyte stack frame.
2216 __ add(as_Register(reg), sp, offset);
2217 }
2218
2219 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2220 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2221 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2222
2223 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2224 return NativeInstruction::instruction_size;
2225 } else {
2226 return 2 * NativeInstruction::instruction_size;
2227 }
2228 }
2229
2230 //=============================================================================
2231
2232 #ifndef PRODUCT
2233 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2234 {
2235 st->print_cr("# MachUEPNode");
2236 if (UseCompressedClassPointers) {
2237 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2238 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2239 st->print_cr("\tcmpw rscratch1, r10");
2240 } else {
2241 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2242 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2243 st->print_cr("\tcmp rscratch1, r10");
2244 }
2245 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2246 }
2247 #endif
2248
2249 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2250 {
2251 __ ic_check(InteriorEntryAlignment);
2252 }
2253
2254 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2255 {
2256 return MachNode::size(ra_);
2257 }
2258
2259 // REQUIRED EMIT CODE
2260
2261 //=============================================================================
2262
2263 // Emit deopt handler code.
2264 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2265 {
2266 // Note that the code buffer's insts_mark is always relative to insts.
2267 // That's why we must use the macroassembler to generate a handler.
2268 address base = __ start_a_stub(size_deopt_handler());
2269 if (base == nullptr) {
2270 ciEnv::current()->record_failure("CodeCache is full");
2271 return 0; // CodeBuffer::expand failed
2272 }
2273
2274 int offset = __ offset();
2275 Label start;
2276 __ bind(start);
2277 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2278
2279 int entry_offset = __ offset();
2280 __ b(start);
2281
2282 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2283 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2284 "out of bounds read in post-call NOP check");
2285 __ end_a_stub();
2286 return entry_offset;
2287 }
2288
2289 // REQUIRED MATCHER CODE
2290
2291 //=============================================================================
2292
2293 bool Matcher::match_rule_supported(int opcode) {
2294 if (!has_match_rule(opcode))
2295 return false;
2296
2297 switch (opcode) {
2298 case Op_OnSpinWait:
2299 return VM_Version::supports_on_spin_wait();
2300 case Op_CacheWB:
2301 case Op_CacheWBPreSync:
2302 case Op_CacheWBPostSync:
2303 if (!VM_Version::supports_data_cache_line_flush()) {
2304 return false;
2305 }
2306 break;
2307 case Op_ExpandBits:
2308 case Op_CompressBits:
2309 if (!VM_Version::supports_svebitperm()) {
2310 return false;
2311 }
2312 break;
2313 case Op_FmaF:
2314 case Op_FmaD:
2315 case Op_FmaVF:
2316 case Op_FmaVD:
2317 if (!UseFMA) {
2318 return false;
2319 }
2320 break;
2321 case Op_FmaHF:
2322 // UseFMA flag also needs to be checked along with FEAT_FP16
2323 if (!UseFMA || !is_feat_fp16_supported()) {
2324 return false;
2325 }
2326 break;
2327 case Op_AddHF:
2328 case Op_SubHF:
2329 case Op_MulHF:
2330 case Op_DivHF:
2331 case Op_MinHF:
2332 case Op_MaxHF:
2333 case Op_SqrtHF:
2334 // Half-precision floating point scalar operations require FEAT_FP16
2335 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2336 // features are supported.
2337 if (!is_feat_fp16_supported()) {
2338 return false;
2339 }
2340 break;
2341 }
2342
2343 return true; // Per default match rules are supported.
2344 }
2345
2346 const RegMask* Matcher::predicate_reg_mask(void) {
2347 return &_PR_REG_mask;
2348 }
2349
2350 bool Matcher::supports_vector_calling_convention(void) {
2351 return EnableVectorSupport;
2352 }
2353
2354 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2355 assert(EnableVectorSupport, "sanity");
2356 int lo = V0_num;
2357 int hi = V0_H_num;
2358 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2359 hi = V0_K_num;
2360 }
2361 return OptoRegPair(hi, lo);
2362 }
2363
2364 // Is this branch offset short enough that a short branch can be used?
2365 //
2366 // NOTE: If the platform does not provide any short branch variants, then
2367 // this method should return false for offset 0.
2368 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2369 // The passed offset is relative to address of the branch.
2370
2371 return (-32768 <= offset && offset < 32768);
2372 }
2373
2374 // Vector width in bytes.
2375 int Matcher::vector_width_in_bytes(BasicType bt) {
2376 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2377 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2378 // Minimum 2 values in vector
2379 if (size < 2*type2aelembytes(bt)) size = 0;
2380 // But never < 4
2381 if (size < 4) size = 0;
2382 return size;
2383 }
2384
2385 // Limits on vector size (number of elements) loaded into vector.
2386 int Matcher::max_vector_size(const BasicType bt) {
2387 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2388 }
2389
2390 int Matcher::min_vector_size(const BasicType bt) {
2391 // Usually, the shortest vector length supported by AArch64 ISA and
2392 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2393 // vectors in a few special cases.
2394 int size;
2395 switch(bt) {
2396 case T_BOOLEAN:
2397 // Load/store a vector mask with only 2 elements for vector types
2398 // such as "2I/2F/2L/2D".
2399 size = 2;
2400 break;
2401 case T_BYTE:
2402 // Generate a "4B" vector, to support vector cast between "8B/16B"
2403 // and "4S/4I/4L/4F/4D".
2404 size = 4;
2405 break;
2406 case T_SHORT:
2407 // Generate a "2S" vector, to support vector cast between "4S/8S"
2408 // and "2I/2L/2F/2D".
2409 size = 2;
2410 break;
2411 default:
2412 // Limit the min vector length to 64-bit.
2413 size = 8 / type2aelembytes(bt);
2414 // The number of elements in a vector should be at least 2.
2415 size = MAX2(size, 2);
2416 }
2417
2418 int max_size = max_vector_size(bt);
2419 return MIN2(size, max_size);
2420 }
2421
2422 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2423 return Matcher::max_vector_size(bt);
2424 }
2425
2426 // Actual max scalable vector register length.
2427 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2428 return Matcher::max_vector_size(bt);
2429 }
2430
2431 // Vector ideal reg.
2432 uint Matcher::vector_ideal_reg(int len) {
2433 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2434 return Op_VecA;
2435 }
2436 switch(len) {
2437 // For 16-bit/32-bit mask vector, reuse VecD.
2438 case 2:
2439 case 4:
2440 case 8: return Op_VecD;
2441 case 16: return Op_VecX;
2442 }
2443 ShouldNotReachHere();
2444 return 0;
2445 }
2446
2447 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2448 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2449 switch (ideal_reg) {
2450 case Op_VecA: return new vecAOper();
2451 case Op_VecD: return new vecDOper();
2452 case Op_VecX: return new vecXOper();
2453 }
2454 ShouldNotReachHere();
2455 return nullptr;
2456 }
2457
2458 bool Matcher::is_reg2reg_move(MachNode* m) {
2459 return false;
2460 }
2461
2462 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2463 return false;
2464 }
2465
2466 bool Matcher::is_generic_vector(MachOper* opnd) {
2467 return opnd->opcode() == VREG;
2468 }
2469
2470 #ifdef ASSERT
2471 // Return whether or not this register is ever used as an argument.
2472 bool Matcher::can_be_java_arg(int reg)
2473 {
2474 return
2475 reg == R0_num || reg == R0_H_num ||
2476 reg == R1_num || reg == R1_H_num ||
2477 reg == R2_num || reg == R2_H_num ||
2478 reg == R3_num || reg == R3_H_num ||
2479 reg == R4_num || reg == R4_H_num ||
2480 reg == R5_num || reg == R5_H_num ||
2481 reg == R6_num || reg == R6_H_num ||
2482 reg == R7_num || reg == R7_H_num ||
2483 reg == V0_num || reg == V0_H_num ||
2484 reg == V1_num || reg == V1_H_num ||
2485 reg == V2_num || reg == V2_H_num ||
2486 reg == V3_num || reg == V3_H_num ||
2487 reg == V4_num || reg == V4_H_num ||
2488 reg == V5_num || reg == V5_H_num ||
2489 reg == V6_num || reg == V6_H_num ||
2490 reg == V7_num || reg == V7_H_num;
2491 }
2492 #endif
2493
2494 uint Matcher::int_pressure_limit()
2495 {
2496 // JDK-8183543: When taking the number of available registers as int
2497 // register pressure threshold, the jtreg test:
2498 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2499 // failed due to C2 compilation failure with
2500 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2501 //
2502 // A derived pointer is live at CallNode and then is flagged by RA
2503 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2504 // derived pointers and lastly fail to spill after reaching maximum
2505 // number of iterations. Lowering the default pressure threshold to
2506 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2507 // a high register pressure area of the code so that split_DEF can
2508 // generate DefinitionSpillCopy for the derived pointer.
2509 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2510 if (!PreserveFramePointer) {
2511 // When PreserveFramePointer is off, frame pointer is allocatable,
2512 // but different from other SOC registers, it is excluded from
2513 // fatproj's mask because its save type is No-Save. Decrease 1 to
2514 // ensure high pressure at fatproj when PreserveFramePointer is off.
2515 // See check_pressure_at_fatproj().
2516 default_int_pressure_threshold--;
2517 }
2518 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2519 }
2520
2521 uint Matcher::float_pressure_limit()
2522 {
2523 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2524 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2525 }
2526
2527 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2528 return false;
2529 }
2530
2531 const RegMask& Matcher::divI_proj_mask() {
2532 ShouldNotReachHere();
2533 return RegMask::EMPTY;
2534 }
2535
2536 // Register for MODI projection of divmodI.
2537 const RegMask& Matcher::modI_proj_mask() {
2538 ShouldNotReachHere();
2539 return RegMask::EMPTY;
2540 }
2541
2542 // Register for DIVL projection of divmodL.
2543 const RegMask& Matcher::divL_proj_mask() {
2544 ShouldNotReachHere();
2545 return RegMask::EMPTY;
2546 }
2547
2548 // Register for MODL projection of divmodL.
2549 const RegMask& Matcher::modL_proj_mask() {
2550 ShouldNotReachHere();
2551 return RegMask::EMPTY;
2552 }
2553
2554 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2555 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2556 Node* u = addp->fast_out(i);
2557 if (u->is_LoadStore()) {
2558 // On AArch64, LoadStoreNodes (i.e. compare and swap
2559 // instructions) only take register indirect as an operand, so
2560 // any attempt to use an AddPNode as an input to a LoadStoreNode
2561 // must fail.
2562 return false;
2563 }
2564 if (u->is_Mem()) {
2565 int opsize = u->as_Mem()->memory_size();
2566 assert(opsize > 0, "unexpected memory operand size");
2567 if (u->as_Mem()->memory_size() != (1<<shift)) {
2568 return false;
2569 }
2570 }
2571 }
2572 return true;
2573 }
2574
2575 // Convert BoolTest condition to Assembler condition.
2576 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2577 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2578 Assembler::Condition result;
2579 switch(cond) {
2580 case BoolTest::eq:
2581 result = Assembler::EQ; break;
2582 case BoolTest::ne:
2583 result = Assembler::NE; break;
2584 case BoolTest::le:
2585 result = Assembler::LE; break;
2586 case BoolTest::ge:
2587 result = Assembler::GE; break;
2588 case BoolTest::lt:
2589 result = Assembler::LT; break;
2590 case BoolTest::gt:
2591 result = Assembler::GT; break;
2592 case BoolTest::ule:
2593 result = Assembler::LS; break;
2594 case BoolTest::uge:
2595 result = Assembler::HS; break;
2596 case BoolTest::ult:
2597 result = Assembler::LO; break;
2598 case BoolTest::ugt:
2599 result = Assembler::HI; break;
2600 case BoolTest::overflow:
2601 result = Assembler::VS; break;
2602 case BoolTest::no_overflow:
2603 result = Assembler::VC; break;
2604 default:
2605 ShouldNotReachHere();
2606 return Assembler::Condition(-1);
2607 }
2608
2609 // Check conversion
2610 if (cond & BoolTest::unsigned_compare) {
2611 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2612 } else {
2613 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2614 }
2615
2616 return result;
2617 }
2618
2619 // Binary src (Replicate con)
2620 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2621 if (n == nullptr || m == nullptr) {
2622 return false;
2623 }
2624
2625 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2626 return false;
2627 }
2628
2629 Node* imm_node = m->in(1);
2630 if (!imm_node->is_Con()) {
2631 return false;
2632 }
2633
2634 const Type* t = imm_node->bottom_type();
2635 if (!(t->isa_int() || t->isa_long())) {
2636 return false;
2637 }
2638
2639 switch (n->Opcode()) {
2640 case Op_AndV:
2641 case Op_OrV:
2642 case Op_XorV: {
2643 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2644 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2645 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2646 }
2647 case Op_AddVB:
2648 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2649 case Op_AddVS:
2650 case Op_AddVI:
2651 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2652 case Op_AddVL:
2653 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2654 default:
2655 return false;
2656 }
2657 }
2658
2659 // (XorV src (Replicate m1))
2660 // (XorVMask src (MaskAll m1))
2661 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2662 if (n != nullptr && m != nullptr) {
2663 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2664 VectorNode::is_all_ones_vector(m);
2665 }
2666 return false;
2667 }
2668
2669 // Should the matcher clone input 'm' of node 'n'?
2670 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2671 if (is_vshift_con_pattern(n, m) ||
2672 is_vector_bitwise_not_pattern(n, m) ||
2673 is_valid_sve_arith_imm_pattern(n, m) ||
2674 is_encode_and_store_pattern(n, m)) {
2675 mstack.push(m, Visit);
2676 return true;
2677 }
2678 return false;
2679 }
2680
2681 // Should the Matcher clone shifts on addressing modes, expecting them
2682 // to be subsumed into complex addressing expressions or compute them
2683 // into registers?
2684 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2685
2686 // Loads and stores with indirect memory input (e.g., volatile loads and
2687 // stores) do not subsume the input into complex addressing expressions. If
2688 // the addressing expression is input to at least one such load or store, do
2689 // not clone the addressing expression. Query needs_acquiring_load and
2690 // needs_releasing_store as a proxy for indirect memory input, as it is not
2691 // possible to directly query for indirect memory input at this stage.
2692 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2693 Node* n = m->fast_out(i);
2694 if (n->is_Load() && needs_acquiring_load(n)) {
2695 return false;
2696 }
2697 if (n->is_Store() && needs_releasing_store(n)) {
2698 return false;
2699 }
2700 }
2701
2702 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2703 return true;
2704 }
2705
2706 Node *off = m->in(AddPNode::Offset);
2707 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2708 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2709 // Are there other uses besides address expressions?
2710 !is_visited(off)) {
2711 address_visited.set(off->_idx); // Flag as address_visited
2712 mstack.push(off->in(2), Visit);
2713 Node *conv = off->in(1);
2714 if (conv->Opcode() == Op_ConvI2L &&
2715 // Are there other uses besides address expressions?
2716 !is_visited(conv)) {
2717 address_visited.set(conv->_idx); // Flag as address_visited
2718 mstack.push(conv->in(1), Pre_Visit);
2719 } else {
2720 mstack.push(conv, Pre_Visit);
2721 }
2722 address_visited.test_set(m->_idx); // Flag as address_visited
2723 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2724 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2725 return true;
2726 } else if (off->Opcode() == Op_ConvI2L &&
2727 // Are there other uses besides address expressions?
2728 !is_visited(off)) {
2729 address_visited.test_set(m->_idx); // Flag as address_visited
2730 address_visited.set(off->_idx); // Flag as address_visited
2731 mstack.push(off->in(1), Pre_Visit);
2732 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2733 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2734 return true;
2735 }
2736 return false;
2737 }
2738
2739 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2740 { \
2741 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2742 guarantee(DISP == 0, "mode not permitted for volatile"); \
2743 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2744 __ INSN(REG, as_Register(BASE)); \
2745 }
2746
2747
2748 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2749 {
2750 Address::extend scale;
2751
2752 // Hooboy, this is fugly. We need a way to communicate to the
2753 // encoder that the index needs to be sign extended, so we have to
2754 // enumerate all the cases.
2755 switch (opcode) {
2756 case INDINDEXSCALEDI2L:
2757 case INDINDEXSCALEDI2LN:
2758 case INDINDEXI2L:
2759 case INDINDEXI2LN:
2760 scale = Address::sxtw(size);
2761 break;
2762 default:
2763 scale = Address::lsl(size);
2764 }
2765
2766 if (index == -1) {
2767 return Address(base, disp);
2768 } else {
2769 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2770 return Address(base, as_Register(index), scale);
2771 }
2772 }
2773
2774
2775 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2776 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2777 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2778 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2779 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2780
2781 // Used for all non-volatile memory accesses. The use of
2782 // $mem->opcode() to discover whether this pattern uses sign-extended
2783 // offsets is something of a kludge.
2784 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2785 Register reg, int opcode,
2786 Register base, int index, int scale, int disp,
2787 int size_in_memory)
2788 {
2789 Address addr = mem2address(opcode, base, index, scale, disp);
2790 if (addr.getMode() == Address::base_plus_offset) {
2791 /* Fix up any out-of-range offsets. */
2792 assert_different_registers(rscratch1, base);
2793 assert_different_registers(rscratch1, reg);
2794 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2795 }
2796 (masm->*insn)(reg, addr);
2797 }
2798
2799 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2800 FloatRegister reg, int opcode,
2801 Register base, int index, int size, int disp,
2802 int size_in_memory)
2803 {
2804 Address::extend scale;
2805
2806 switch (opcode) {
2807 case INDINDEXSCALEDI2L:
2808 case INDINDEXSCALEDI2LN:
2809 scale = Address::sxtw(size);
2810 break;
2811 default:
2812 scale = Address::lsl(size);
2813 }
2814
2815 if (index == -1) {
2816 // Fix up any out-of-range offsets.
2817 assert_different_registers(rscratch1, base);
2818 Address addr = Address(base, disp);
2819 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2820 (masm->*insn)(reg, addr);
2821 } else {
2822 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2823 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2824 }
2825 }
2826
2827 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2828 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2829 int opcode, Register base, int index, int size, int disp)
2830 {
2831 if (index == -1) {
2832 (masm->*insn)(reg, T, Address(base, disp));
2833 } else {
2834 assert(disp == 0, "unsupported address mode");
2835 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2836 }
2837 }
2838
2839 %}
2840
2841
2842
2843 //----------ENCODING BLOCK-----------------------------------------------------
2844 // This block specifies the encoding classes used by the compiler to
2845 // output byte streams. Encoding classes are parameterized macros
2846 // used by Machine Instruction Nodes in order to generate the bit
2847 // encoding of the instruction. Operands specify their base encoding
2848 // interface with the interface keyword. There are currently
2849 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2850 // COND_INTER. REG_INTER causes an operand to generate a function
2851 // which returns its register number when queried. CONST_INTER causes
2852 // an operand to generate a function which returns the value of the
2853 // constant when queried. MEMORY_INTER causes an operand to generate
2854 // four functions which return the Base Register, the Index Register,
2855 // the Scale Value, and the Offset Value of the operand when queried.
2856 // COND_INTER causes an operand to generate six functions which return
2857 // the encoding code (ie - encoding bits for the instruction)
2858 // associated with each basic boolean condition for a conditional
2859 // instruction.
2860 //
2861 // Instructions specify two basic values for encoding. Again, a
2862 // function is available to check if the constant displacement is an
2863 // oop. They use the ins_encode keyword to specify their encoding
2864 // classes (which must be a sequence of enc_class names, and their
2865 // parameters, specified in the encoding block), and they use the
2866 // opcode keyword to specify, in order, their primary, secondary, and
2867 // tertiary opcode. Only the opcode sections which a particular
2868 // instruction needs for encoding need to be specified.
2869 encode %{
2870 // Build emit functions for each basic byte or larger field in the
2871 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2872 // from C++ code in the enc_class source block. Emit functions will
2873 // live in the main source block for now. In future, we can
2874 // generalize this by adding a syntax that specifies the sizes of
2875 // fields in an order, so that the adlc can build the emit functions
2876 // automagically
2877
2878 // catch all for unimplemented encodings
2879 enc_class enc_unimplemented %{
2880 __ unimplemented("C2 catch all");
2881 %}
2882
2883 // BEGIN Non-volatile memory access
2884
2885 // This encoding class is generated automatically from ad_encode.m4.
2886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2887 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2888 Register dst_reg = as_Register($dst$$reg);
2889 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2890 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2891 %}
2892
2893 // This encoding class is generated automatically from ad_encode.m4.
2894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2895 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2896 Register dst_reg = as_Register($dst$$reg);
2897 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2899 %}
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2904 Register dst_reg = as_Register($dst$$reg);
2905 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2912 Register dst_reg = as_Register($dst$$reg);
2913 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2920 Register dst_reg = as_Register($dst$$reg);
2921 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2928 Register dst_reg = as_Register($dst$$reg);
2929 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2936 Register dst_reg = as_Register($dst$$reg);
2937 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2944 Register dst_reg = as_Register($dst$$reg);
2945 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2952 Register dst_reg = as_Register($dst$$reg);
2953 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2960 Register dst_reg = as_Register($dst$$reg);
2961 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2968 Register dst_reg = as_Register($dst$$reg);
2969 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2976 Register dst_reg = as_Register($dst$$reg);
2977 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2978 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2979 %}
2980
2981 // This encoding class is generated automatically from ad_encode.m4.
2982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2983 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2984 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2985 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2986 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2987 %}
2988
2989 // This encoding class is generated automatically from ad_encode.m4.
2990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2991 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2992 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2993 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3000 Register src_reg = as_Register($src$$reg);
3001 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3002 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3003 %}
3004
3005 // This encoding class is generated automatically from ad_encode.m4.
3006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3007 enc_class aarch64_enc_strb0(memory1 mem) %{
3008 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3009 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3010 %}
3011
3012 // This encoding class is generated automatically from ad_encode.m4.
3013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3014 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3015 Register src_reg = as_Register($src$$reg);
3016 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3017 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3018 %}
3019
3020 // This encoding class is generated automatically from ad_encode.m4.
3021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3022 enc_class aarch64_enc_strh0(memory2 mem) %{
3023 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3024 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3025 %}
3026
3027 // This encoding class is generated automatically from ad_encode.m4.
3028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3029 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3030 Register src_reg = as_Register($src$$reg);
3031 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3032 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3033 %}
3034
3035 // This encoding class is generated automatically from ad_encode.m4.
3036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3037 enc_class aarch64_enc_strw0(memory4 mem) %{
3038 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3039 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3040 %}
3041
3042 // This encoding class is generated automatically from ad_encode.m4.
3043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3044 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3045 Register src_reg = as_Register($src$$reg);
3046 // we sometimes get asked to store the stack pointer into the
3047 // current thread -- we cannot do that directly on AArch64
3048 if (src_reg == r31_sp) {
3049 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3050 __ mov(rscratch2, sp);
3051 src_reg = rscratch2;
3052 }
3053 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3054 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3055 %}
3056
3057 // This encoding class is generated automatically from ad_encode.m4.
3058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3059 enc_class aarch64_enc_str0(memory8 mem) %{
3060 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3061 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3062 %}
3063
3064 // This encoding class is generated automatically from ad_encode.m4.
3065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3066 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3067 FloatRegister src_reg = as_FloatRegister($src$$reg);
3068 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3069 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3070 %}
3071
3072 // This encoding class is generated automatically from ad_encode.m4.
3073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3074 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3075 FloatRegister src_reg = as_FloatRegister($src$$reg);
3076 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3077 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3078 %}
3079
3080 // This encoding class is generated automatically from ad_encode.m4.
3081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3082 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3083 __ membar(Assembler::StoreStore);
3084 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3085 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3086 %}
3087
3088 // END Non-volatile memory access
3089
3090 // Vector loads and stores
3091 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3092 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3093 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3094 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3095 %}
3096
3097 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3098 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3099 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3100 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3101 %}
3102
3103 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3104 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3105 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3106 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3107 %}
3108
3109 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3110 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3111 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3112 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3113 %}
3114
3115 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3116 FloatRegister src_reg = as_FloatRegister($src$$reg);
3117 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3118 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3119 %}
3120
3121 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3122 FloatRegister src_reg = as_FloatRegister($src$$reg);
3123 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3124 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3125 %}
3126
3127 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3128 FloatRegister src_reg = as_FloatRegister($src$$reg);
3129 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3130 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3131 %}
3132
3133 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3134 FloatRegister src_reg = as_FloatRegister($src$$reg);
3135 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3136 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3137 %}
3138
3139 // volatile loads and stores
3140
3141 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3142 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3143 rscratch1, stlrb);
3144 %}
3145
3146 enc_class aarch64_enc_stlrb0(memory mem) %{
3147 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3148 rscratch1, stlrb);
3149 %}
3150
3151 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3152 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, stlrh);
3154 %}
3155
3156 enc_class aarch64_enc_stlrh0(memory mem) %{
3157 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, stlrh);
3159 %}
3160
3161 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3162 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3163 rscratch1, stlrw);
3164 %}
3165
3166 enc_class aarch64_enc_stlrw0(memory mem) %{
3167 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3168 rscratch1, stlrw);
3169 %}
3170
3171 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3172 Register dst_reg = as_Register($dst$$reg);
3173 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3174 rscratch1, ldarb);
3175 __ sxtbw(dst_reg, dst_reg);
3176 %}
3177
3178 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3179 Register dst_reg = as_Register($dst$$reg);
3180 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldarb);
3182 __ sxtb(dst_reg, dst_reg);
3183 %}
3184
3185 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3186 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3187 rscratch1, ldarb);
3188 %}
3189
3190 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3191 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3192 rscratch1, ldarb);
3193 %}
3194
3195 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3196 Register dst_reg = as_Register($dst$$reg);
3197 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, ldarh);
3199 __ sxthw(dst_reg, dst_reg);
3200 %}
3201
3202 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3203 Register dst_reg = as_Register($dst$$reg);
3204 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3205 rscratch1, ldarh);
3206 __ sxth(dst_reg, dst_reg);
3207 %}
3208
3209 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3210 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3211 rscratch1, ldarh);
3212 %}
3213
3214 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3215 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3216 rscratch1, ldarh);
3217 %}
3218
3219 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3220 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, ldarw);
3222 %}
3223
3224 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3225 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldarw);
3227 %}
3228
3229 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3230 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3231 rscratch1, ldar);
3232 %}
3233
3234 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3235 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3236 rscratch1, ldarw);
3237 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3238 %}
3239
3240 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3241 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3242 rscratch1, ldar);
3243 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3244 %}
3245
3246 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3247 Register src_reg = as_Register($src$$reg);
3248 // we sometimes get asked to store the stack pointer into the
3249 // current thread -- we cannot do that directly on AArch64
3250 if (src_reg == r31_sp) {
3251 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3252 __ mov(rscratch2, sp);
3253 src_reg = rscratch2;
3254 }
3255 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3256 rscratch1, stlr);
3257 %}
3258
3259 enc_class aarch64_enc_stlr0(memory mem) %{
3260 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3261 rscratch1, stlr);
3262 %}
3263
3264 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3265 {
3266 FloatRegister src_reg = as_FloatRegister($src$$reg);
3267 __ fmovs(rscratch2, src_reg);
3268 }
3269 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3270 rscratch1, stlrw);
3271 %}
3272
3273 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3274 {
3275 FloatRegister src_reg = as_FloatRegister($src$$reg);
3276 __ fmovd(rscratch2, src_reg);
3277 }
3278 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3279 rscratch1, stlr);
3280 %}
3281
3282 // synchronized read/update encodings
3283
3284 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3285 Register dst_reg = as_Register($dst$$reg);
3286 Register base = as_Register($mem$$base);
3287 int index = $mem$$index;
3288 int scale = $mem$$scale;
3289 int disp = $mem$$disp;
3290 if (index == -1) {
3291 if (disp != 0) {
3292 __ lea(rscratch1, Address(base, disp));
3293 __ ldaxr(dst_reg, rscratch1);
3294 } else {
3295 // TODO
3296 // should we ever get anything other than this case?
3297 __ ldaxr(dst_reg, base);
3298 }
3299 } else {
3300 Register index_reg = as_Register(index);
3301 if (disp == 0) {
3302 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3303 __ ldaxr(dst_reg, rscratch1);
3304 } else {
3305 __ lea(rscratch1, Address(base, disp));
3306 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3307 __ ldaxr(dst_reg, rscratch1);
3308 }
3309 }
3310 %}
3311
3312 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3313 Register src_reg = as_Register($src$$reg);
3314 Register base = as_Register($mem$$base);
3315 int index = $mem$$index;
3316 int scale = $mem$$scale;
3317 int disp = $mem$$disp;
3318 if (index == -1) {
3319 if (disp != 0) {
3320 __ lea(rscratch2, Address(base, disp));
3321 __ stlxr(rscratch1, src_reg, rscratch2);
3322 } else {
3323 // TODO
3324 // should we ever get anything other than this case?
3325 __ stlxr(rscratch1, src_reg, base);
3326 }
3327 } else {
3328 Register index_reg = as_Register(index);
3329 if (disp == 0) {
3330 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3331 __ stlxr(rscratch1, src_reg, rscratch2);
3332 } else {
3333 __ lea(rscratch2, Address(base, disp));
3334 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3335 __ stlxr(rscratch1, src_reg, rscratch2);
3336 }
3337 }
3338 __ cmpw(rscratch1, zr);
3339 %}
3340
3341 // prefetch encodings
3342
3343 enc_class aarch64_enc_prefetchw(memory mem) %{
3344 Register base = as_Register($mem$$base);
3345 int index = $mem$$index;
3346 int scale = $mem$$scale;
3347 int disp = $mem$$disp;
3348 if (index == -1) {
3349 // Fix up any out-of-range offsets.
3350 assert_different_registers(rscratch1, base);
3351 Address addr = Address(base, disp);
3352 addr = __ legitimize_address(addr, 8, rscratch1);
3353 __ prfm(addr, PSTL1KEEP);
3354 } else {
3355 Register index_reg = as_Register(index);
3356 if (disp == 0) {
3357 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3358 } else {
3359 __ lea(rscratch1, Address(base, disp));
3360 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3361 }
3362 }
3363 %}
3364
3365 // mov encodings
3366
3367 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3368 uint32_t con = (uint32_t)$src$$constant;
3369 Register dst_reg = as_Register($dst$$reg);
3370 if (con == 0) {
3371 __ movw(dst_reg, zr);
3372 } else {
3373 __ movw(dst_reg, con);
3374 }
3375 %}
3376
3377 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3378 Register dst_reg = as_Register($dst$$reg);
3379 uint64_t con = (uint64_t)$src$$constant;
3380 if (con == 0) {
3381 __ mov(dst_reg, zr);
3382 } else {
3383 __ mov(dst_reg, con);
3384 }
3385 %}
3386
3387 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3388 Register dst_reg = as_Register($dst$$reg);
3389 address con = (address)$src$$constant;
3390 if (con == nullptr || con == (address)1) {
3391 ShouldNotReachHere();
3392 } else {
3393 relocInfo::relocType rtype = $src->constant_reloc();
3394 if (rtype == relocInfo::oop_type) {
3395 __ movoop(dst_reg, (jobject)con);
3396 } else if (rtype == relocInfo::metadata_type) {
3397 __ mov_metadata(dst_reg, (Metadata*)con);
3398 } else {
3399 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3400 // load fake address constants using a normal move
3401 if (! __ is_valid_AArch64_address(con) ||
3402 con < (address)(uintptr_t)os::vm_page_size()) {
3403 __ mov(dst_reg, con);
3404 } else {
3405 // no reloc so just use adrp and add
3406 uint64_t offset;
3407 __ adrp(dst_reg, con, offset);
3408 __ add(dst_reg, dst_reg, offset);
3409 }
3410 }
3411 }
3412 %}
3413
3414 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3415 Register dst_reg = as_Register($dst$$reg);
3416 __ mov(dst_reg, zr);
3417 %}
3418
3419 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3420 Register dst_reg = as_Register($dst$$reg);
3421 __ mov(dst_reg, (uint64_t)1);
3422 %}
3423
3424 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3425 Register dst_reg = as_Register($dst$$reg);
3426 address con = (address)$src$$constant;
3427 if (con == nullptr) {
3428 ShouldNotReachHere();
3429 } else {
3430 relocInfo::relocType rtype = $src->constant_reloc();
3431 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3432 __ set_narrow_oop(dst_reg, (jobject)con);
3433 }
3434 %}
3435
3436 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3437 Register dst_reg = as_Register($dst$$reg);
3438 __ mov(dst_reg, zr);
3439 %}
3440
3441 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3442 Register dst_reg = as_Register($dst$$reg);
3443 address con = (address)$src$$constant;
3444 if (con == nullptr) {
3445 ShouldNotReachHere();
3446 } else {
3447 relocInfo::relocType rtype = $src->constant_reloc();
3448 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3449 __ set_narrow_klass(dst_reg, (Klass *)con);
3450 }
3451 %}
3452
3453 // arithmetic encodings
3454
3455 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3456 Register dst_reg = as_Register($dst$$reg);
3457 Register src_reg = as_Register($src1$$reg);
3458 int32_t con = (int32_t)$src2$$constant;
3459 // add has primary == 0, subtract has primary == 1
3460 if ($primary) { con = -con; }
3461 if (con < 0) {
3462 __ subw(dst_reg, src_reg, -con);
3463 } else {
3464 __ addw(dst_reg, src_reg, con);
3465 }
3466 %}
3467
3468 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3469 Register dst_reg = as_Register($dst$$reg);
3470 Register src_reg = as_Register($src1$$reg);
3471 int32_t con = (int32_t)$src2$$constant;
3472 // add has primary == 0, subtract has primary == 1
3473 if ($primary) { con = -con; }
3474 if (con < 0) {
3475 __ sub(dst_reg, src_reg, -con);
3476 } else {
3477 __ add(dst_reg, src_reg, con);
3478 }
3479 %}
3480
3481 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3482 Register dst_reg = as_Register($dst$$reg);
3483 Register src1_reg = as_Register($src1$$reg);
3484 Register src2_reg = as_Register($src2$$reg);
3485 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3486 %}
3487
3488 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3489 Register dst_reg = as_Register($dst$$reg);
3490 Register src1_reg = as_Register($src1$$reg);
3491 Register src2_reg = as_Register($src2$$reg);
3492 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3493 %}
3494
3495 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3496 Register dst_reg = as_Register($dst$$reg);
3497 Register src1_reg = as_Register($src1$$reg);
3498 Register src2_reg = as_Register($src2$$reg);
3499 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3500 %}
3501
3502 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3503 Register dst_reg = as_Register($dst$$reg);
3504 Register src1_reg = as_Register($src1$$reg);
3505 Register src2_reg = as_Register($src2$$reg);
3506 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3507 %}
3508
3509 // compare instruction encodings
3510
3511 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3512 Register reg1 = as_Register($src1$$reg);
3513 Register reg2 = as_Register($src2$$reg);
3514 __ cmpw(reg1, reg2);
3515 %}
3516
3517 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3518 Register reg = as_Register($src1$$reg);
3519 int32_t val = $src2$$constant;
3520 if (val >= 0) {
3521 __ subsw(zr, reg, val);
3522 } else {
3523 __ addsw(zr, reg, -val);
3524 }
3525 %}
3526
3527 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3528 Register reg1 = as_Register($src1$$reg);
3529 uint32_t val = (uint32_t)$src2$$constant;
3530 __ movw(rscratch1, val);
3531 __ cmpw(reg1, rscratch1);
3532 %}
3533
3534 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3535 Register reg1 = as_Register($src1$$reg);
3536 Register reg2 = as_Register($src2$$reg);
3537 __ cmp(reg1, reg2);
3538 %}
3539
3540 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3541 Register reg = as_Register($src1$$reg);
3542 int64_t val = $src2$$constant;
3543 if (val >= 0) {
3544 __ subs(zr, reg, val);
3545 } else if (val != -val) {
3546 __ adds(zr, reg, -val);
3547 } else {
3548 // aargh, Long.MIN_VALUE is a special case
3549 __ orr(rscratch1, zr, (uint64_t)val);
3550 __ subs(zr, reg, rscratch1);
3551 }
3552 %}
3553
3554 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3555 Register reg1 = as_Register($src1$$reg);
3556 uint64_t val = (uint64_t)$src2$$constant;
3557 __ mov(rscratch1, val);
3558 __ cmp(reg1, rscratch1);
3559 %}
3560
3561 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3562 Register reg1 = as_Register($src1$$reg);
3563 Register reg2 = as_Register($src2$$reg);
3564 __ cmp(reg1, reg2);
3565 %}
3566
3567 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3568 Register reg1 = as_Register($src1$$reg);
3569 Register reg2 = as_Register($src2$$reg);
3570 __ cmpw(reg1, reg2);
3571 %}
3572
3573 enc_class aarch64_enc_testp(iRegP src) %{
3574 Register reg = as_Register($src$$reg);
3575 __ cmp(reg, zr);
3576 %}
3577
3578 enc_class aarch64_enc_testn(iRegN src) %{
3579 Register reg = as_Register($src$$reg);
3580 __ cmpw(reg, zr);
3581 %}
3582
3583 enc_class aarch64_enc_b(label lbl) %{
3584 Label *L = $lbl$$label;
3585 __ b(*L);
3586 %}
3587
3588 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3589 Label *L = $lbl$$label;
3590 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3591 %}
3592
3593 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3594 Label *L = $lbl$$label;
3595 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3596 %}
3597
3598 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3599 %{
3600 Register sub_reg = as_Register($sub$$reg);
3601 Register super_reg = as_Register($super$$reg);
3602 Register temp_reg = as_Register($temp$$reg);
3603 Register result_reg = as_Register($result$$reg);
3604
3605 Label miss;
3606 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3607 nullptr, &miss,
3608 /*set_cond_codes:*/ true);
3609 if ($primary) {
3610 __ mov(result_reg, zr);
3611 }
3612 __ bind(miss);
3613 %}
3614
3615 enc_class aarch64_enc_java_static_call(method meth) %{
3616 address addr = (address)$meth$$method;
3617 address call;
3618 if (!_method) {
3619 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3620 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3621 if (call == nullptr) {
3622 ciEnv::current()->record_failure("CodeCache is full");
3623 return;
3624 }
3625 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3626 // The NOP here is purely to ensure that eliding a call to
3627 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3628 __ nop();
3629 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3630 } else {
3631 int method_index = resolved_method_index(masm);
3632 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3633 : static_call_Relocation::spec(method_index);
3634 call = __ trampoline_call(Address(addr, rspec));
3635 if (call == nullptr) {
3636 ciEnv::current()->record_failure("CodeCache is full");
3637 return;
3638 }
3639 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3640 // Calls of the same statically bound method can share
3641 // a stub to the interpreter.
3642 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3643 } else {
3644 // Emit stub for static call
3645 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3646 if (stub == nullptr) {
3647 ciEnv::current()->record_failure("CodeCache is full");
3648 return;
3649 }
3650 }
3651 }
3652
3653 __ post_call_nop();
3654
3655 // Only non uncommon_trap calls need to reinitialize ptrue.
3656 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3657 __ reinitialize_ptrue();
3658 }
3659 %}
3660
3661 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3662 int method_index = resolved_method_index(masm);
3663 address call = __ ic_call((address)$meth$$method, method_index);
3664 if (call == nullptr) {
3665 ciEnv::current()->record_failure("CodeCache is full");
3666 return;
3667 }
3668 __ post_call_nop();
3669 if (Compile::current()->max_vector_size() > 0) {
3670 __ reinitialize_ptrue();
3671 }
3672 %}
3673
3674 enc_class aarch64_enc_call_epilog() %{
3675 if (VerifyStackAtCalls) {
3676 // Check that stack depth is unchanged: find majik cookie on stack
3677 __ call_Unimplemented();
3678 }
3679 %}
3680
3681 enc_class aarch64_enc_java_to_runtime(method meth) %{
3682 // some calls to generated routines (arraycopy code) are scheduled
3683 // by C2 as runtime calls. if so we can call them using a br (they
3684 // will be in a reachable segment) otherwise we have to use a blr
3685 // which loads the absolute address into a register.
3686 address entry = (address)$meth$$method;
3687 CodeBlob *cb = CodeCache::find_blob(entry);
3688 if (cb) {
3689 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3690 if (call == nullptr) {
3691 ciEnv::current()->record_failure("CodeCache is full");
3692 return;
3693 }
3694 __ post_call_nop();
3695 } else {
3696 Label retaddr;
3697 // Make the anchor frame walkable
3698 __ adr(rscratch2, retaddr);
3699 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3700 __ lea(rscratch1, RuntimeAddress(entry));
3701 __ blr(rscratch1);
3702 __ bind(retaddr);
3703 __ post_call_nop();
3704 }
3705 if (Compile::current()->max_vector_size() > 0) {
3706 __ reinitialize_ptrue();
3707 }
3708 %}
3709
3710 enc_class aarch64_enc_rethrow() %{
3711 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3712 %}
3713
3714 enc_class aarch64_enc_ret() %{
3715 #ifdef ASSERT
3716 if (Compile::current()->max_vector_size() > 0) {
3717 __ verify_ptrue();
3718 }
3719 #endif
3720 __ ret(lr);
3721 %}
3722
3723 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3724 Register target_reg = as_Register($jump_target$$reg);
3725 __ br(target_reg);
3726 %}
3727
3728 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3729 Register target_reg = as_Register($jump_target$$reg);
3730 // exception oop should be in r0
3731 // ret addr has been popped into lr
3732 // callee expects it in r3
3733 __ mov(r3, lr);
3734 __ br(target_reg);
3735 %}
3736
3737 %}
3738
3739 //----------FRAME--------------------------------------------------------------
3740 // Definition of frame structure and management information.
3741 //
3742 // S T A C K L A Y O U T Allocators stack-slot number
3743 // | (to get allocators register number
3744 // G Owned by | | v add OptoReg::stack0())
3745 // r CALLER | |
3746 // o | +--------+ pad to even-align allocators stack-slot
3747 // w V | pad0 | numbers; owned by CALLER
3748 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3749 // h ^ | in | 5
3750 // | | args | 4 Holes in incoming args owned by SELF
3751 // | | | | 3
3752 // | | +--------+
3753 // V | | old out| Empty on Intel, window on Sparc
3754 // | old |preserve| Must be even aligned.
3755 // | SP-+--------+----> Matcher::_old_SP, even aligned
3756 // | | in | 3 area for Intel ret address
3757 // Owned by |preserve| Empty on Sparc.
3758 // SELF +--------+
3759 // | | pad2 | 2 pad to align old SP
3760 // | +--------+ 1
3761 // | | locks | 0
3762 // | +--------+----> OptoReg::stack0(), even aligned
3763 // | | pad1 | 11 pad to align new SP
3764 // | +--------+
3765 // | | | 10
3766 // | | spills | 9 spills
3767 // V | | 8 (pad0 slot for callee)
3768 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3769 // ^ | out | 7
3770 // | | args | 6 Holes in outgoing args owned by CALLEE
3771 // Owned by +--------+
3772 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3773 // | new |preserve| Must be even-aligned.
3774 // | SP-+--------+----> Matcher::_new_SP, even aligned
3775 // | | |
3776 //
3777 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3778 // known from SELF's arguments and the Java calling convention.
3779 // Region 6-7 is determined per call site.
3780 // Note 2: If the calling convention leaves holes in the incoming argument
3781 // area, those holes are owned by SELF. Holes in the outgoing area
3782 // are owned by the CALLEE. Holes should not be necessary in the
3783 // incoming area, as the Java calling convention is completely under
3784 // the control of the AD file. Doubles can be sorted and packed to
3785 // avoid holes. Holes in the outgoing arguments may be necessary for
3786 // varargs C calling conventions.
3787 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3788 // even aligned with pad0 as needed.
3789 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3790 // (the latter is true on Intel but is it false on AArch64?)
3791 // region 6-11 is even aligned; it may be padded out more so that
3792 // the region from SP to FP meets the minimum stack alignment.
3793 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3794 // alignment. Region 11, pad1, may be dynamically extended so that
3795 // SP meets the minimum alignment.
3796
3797 frame %{
3798 // These three registers define part of the calling convention
3799 // between compiled code and the interpreter.
3800
3801 // Inline Cache Register or Method for I2C.
3802 inline_cache_reg(R12);
3803
3804 // Number of stack slots consumed by locking an object
3805 sync_stack_slots(2);
3806
3807 // Compiled code's Frame Pointer
3808 frame_pointer(R31);
3809
3810 // Stack alignment requirement
3811 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3812
3813 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3814 // for calls to C. Supports the var-args backing area for register parms.
3815 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3816
3817 // The after-PROLOG location of the return address. Location of
3818 // return address specifies a type (REG or STACK) and a number
3819 // representing the register number (i.e. - use a register name) or
3820 // stack slot.
3821 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3822 // Otherwise, it is above the locks and verification slot and alignment word
3823 // TODO this may well be correct but need to check why that - 2 is there
3824 // ppc port uses 0 but we definitely need to allow for fixed_slots
3825 // which folds in the space used for monitors
3826 return_addr(STACK - 2 +
3827 align_up((Compile::current()->in_preserve_stack_slots() +
3828 Compile::current()->fixed_slots()),
3829 stack_alignment_in_slots()));
3830
3831 // Location of compiled Java return values. Same as C for now.
3832 return_value
3833 %{
3834 // TODO do we allow ideal_reg == Op_RegN???
3835 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3836 "only return normal values");
3837
3838 static const int lo[Op_RegL + 1] = { // enum name
3839 0, // Op_Node
3840 0, // Op_Set
3841 R0_num, // Op_RegN
3842 R0_num, // Op_RegI
3843 R0_num, // Op_RegP
3844 V0_num, // Op_RegF
3845 V0_num, // Op_RegD
3846 R0_num // Op_RegL
3847 };
3848
3849 static const int hi[Op_RegL + 1] = { // enum name
3850 0, // Op_Node
3851 0, // Op_Set
3852 OptoReg::Bad, // Op_RegN
3853 OptoReg::Bad, // Op_RegI
3854 R0_H_num, // Op_RegP
3855 OptoReg::Bad, // Op_RegF
3856 V0_H_num, // Op_RegD
3857 R0_H_num // Op_RegL
3858 };
3859
3860 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3861 %}
3862 %}
3863
3864 //----------ATTRIBUTES---------------------------------------------------------
3865 //----------Operand Attributes-------------------------------------------------
3866 op_attrib op_cost(1); // Required cost attribute
3867
3868 //----------Instruction Attributes---------------------------------------------
3869 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3870 ins_attrib ins_size(32); // Required size attribute (in bits)
3871 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3872 // a non-matching short branch variant
3873 // of some long branch?
3874 ins_attrib ins_alignment(4); // Required alignment attribute (must
3875 // be a power of 2) specifies the
3876 // alignment that some part of the
3877 // instruction (not necessarily the
3878 // start) requires. If > 1, a
3879 // compute_padding() function must be
3880 // provided for the instruction
3881
3882 // Whether this node is expanded during code emission into a sequence of
3883 // instructions and the first instruction can perform an implicit null check.
3884 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3885
3886 //----------OPERANDS-----------------------------------------------------------
3887 // Operand definitions must precede instruction definitions for correct parsing
3888 // in the ADLC because operands constitute user defined types which are used in
3889 // instruction definitions.
3890
3891 //----------Simple Operands----------------------------------------------------
3892
3893 // Integer operands 32 bit
3894 // 32 bit immediate
3895 operand immI()
3896 %{
3897 match(ConI);
3898
3899 op_cost(0);
3900 format %{ %}
3901 interface(CONST_INTER);
3902 %}
3903
3904 // 32 bit zero
3905 operand immI0()
3906 %{
3907 predicate(n->get_int() == 0);
3908 match(ConI);
3909
3910 op_cost(0);
3911 format %{ %}
3912 interface(CONST_INTER);
3913 %}
3914
3915 // 32 bit unit increment
3916 operand immI_1()
3917 %{
3918 predicate(n->get_int() == 1);
3919 match(ConI);
3920
3921 op_cost(0);
3922 format %{ %}
3923 interface(CONST_INTER);
3924 %}
3925
3926 // 32 bit unit decrement
3927 operand immI_M1()
3928 %{
3929 predicate(n->get_int() == -1);
3930 match(ConI);
3931
3932 op_cost(0);
3933 format %{ %}
3934 interface(CONST_INTER);
3935 %}
3936
3937 // Shift values for add/sub extension shift
3938 operand immIExt()
3939 %{
3940 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3941 match(ConI);
3942
3943 op_cost(0);
3944 format %{ %}
3945 interface(CONST_INTER);
3946 %}
3947
3948 operand immI_gt_1()
3949 %{
3950 predicate(n->get_int() > 1);
3951 match(ConI);
3952
3953 op_cost(0);
3954 format %{ %}
3955 interface(CONST_INTER);
3956 %}
3957
3958 operand immI_le_4()
3959 %{
3960 predicate(n->get_int() <= 4);
3961 match(ConI);
3962
3963 op_cost(0);
3964 format %{ %}
3965 interface(CONST_INTER);
3966 %}
3967
3968 operand immI_16()
3969 %{
3970 predicate(n->get_int() == 16);
3971 match(ConI);
3972
3973 op_cost(0);
3974 format %{ %}
3975 interface(CONST_INTER);
3976 %}
3977
3978 operand immI_24()
3979 %{
3980 predicate(n->get_int() == 24);
3981 match(ConI);
3982
3983 op_cost(0);
3984 format %{ %}
3985 interface(CONST_INTER);
3986 %}
3987
3988 operand immI_32()
3989 %{
3990 predicate(n->get_int() == 32);
3991 match(ConI);
3992
3993 op_cost(0);
3994 format %{ %}
3995 interface(CONST_INTER);
3996 %}
3997
3998 operand immI_48()
3999 %{
4000 predicate(n->get_int() == 48);
4001 match(ConI);
4002
4003 op_cost(0);
4004 format %{ %}
4005 interface(CONST_INTER);
4006 %}
4007
4008 operand immI_56()
4009 %{
4010 predicate(n->get_int() == 56);
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_255()
4019 %{
4020 predicate(n->get_int() == 255);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_65535()
4029 %{
4030 predicate(n->get_int() == 65535);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 operand immI_positive()
4039 %{
4040 predicate(n->get_int() > 0);
4041 match(ConI);
4042
4043 op_cost(0);
4044 format %{ %}
4045 interface(CONST_INTER);
4046 %}
4047
4048 // BoolTest condition for signed compare
4049 operand immI_cmp_cond()
4050 %{
4051 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4052 match(ConI);
4053
4054 op_cost(0);
4055 format %{ %}
4056 interface(CONST_INTER);
4057 %}
4058
4059 // BoolTest condition for unsigned compare
4060 operand immI_cmpU_cond()
4061 %{
4062 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4063 match(ConI);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 operand immL_255()
4071 %{
4072 predicate(n->get_long() == 255L);
4073 match(ConL);
4074
4075 op_cost(0);
4076 format %{ %}
4077 interface(CONST_INTER);
4078 %}
4079
4080 operand immL_65535()
4081 %{
4082 predicate(n->get_long() == 65535L);
4083 match(ConL);
4084
4085 op_cost(0);
4086 format %{ %}
4087 interface(CONST_INTER);
4088 %}
4089
4090 operand immL_4294967295()
4091 %{
4092 predicate(n->get_long() == 4294967295L);
4093 match(ConL);
4094
4095 op_cost(0);
4096 format %{ %}
4097 interface(CONST_INTER);
4098 %}
4099
4100 operand immL_bitmask()
4101 %{
4102 predicate((n->get_long() != 0)
4103 && ((n->get_long() & 0xc000000000000000l) == 0)
4104 && is_power_of_2(n->get_long() + 1));
4105 match(ConL);
4106
4107 op_cost(0);
4108 format %{ %}
4109 interface(CONST_INTER);
4110 %}
4111
4112 operand immI_bitmask()
4113 %{
4114 predicate((n->get_int() != 0)
4115 && ((n->get_int() & 0xc0000000) == 0)
4116 && is_power_of_2(n->get_int() + 1));
4117 match(ConI);
4118
4119 op_cost(0);
4120 format %{ %}
4121 interface(CONST_INTER);
4122 %}
4123
4124 operand immL_positive_bitmaskI()
4125 %{
4126 predicate((n->get_long() != 0)
4127 && ((julong)n->get_long() < 0x80000000ULL)
4128 && is_power_of_2(n->get_long() + 1));
4129 match(ConL);
4130
4131 op_cost(0);
4132 format %{ %}
4133 interface(CONST_INTER);
4134 %}
4135
4136 // Scale values for scaled offset addressing modes (up to long but not quad)
4137 operand immIScale()
4138 %{
4139 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4140 match(ConI);
4141
4142 op_cost(0);
4143 format %{ %}
4144 interface(CONST_INTER);
4145 %}
4146
4147 // 5 bit signed integer
4148 operand immI5()
4149 %{
4150 predicate(Assembler::is_simm(n->get_int(), 5));
4151 match(ConI);
4152
4153 op_cost(0);
4154 format %{ %}
4155 interface(CONST_INTER);
4156 %}
4157
4158 // 7 bit unsigned integer
4159 operand immIU7()
4160 %{
4161 predicate(Assembler::is_uimm(n->get_int(), 7));
4162 match(ConI);
4163
4164 op_cost(0);
4165 format %{ %}
4166 interface(CONST_INTER);
4167 %}
4168
4169 // Offset for scaled or unscaled immediate loads and stores
4170 operand immIOffset()
4171 %{
4172 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4173 match(ConI);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 operand immIOffset1()
4181 %{
4182 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4183 match(ConI);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 operand immIOffset2()
4191 %{
4192 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4193 match(ConI);
4194
4195 op_cost(0);
4196 format %{ %}
4197 interface(CONST_INTER);
4198 %}
4199
4200 operand immIOffset4()
4201 %{
4202 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 operand immIOffset8()
4211 %{
4212 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immIOffset16()
4221 %{
4222 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4223 match(ConI);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immLOffset()
4231 %{
4232 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4233 match(ConL);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immLoffset1()
4241 %{
4242 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4243 match(ConL);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immLoffset2()
4251 %{
4252 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4253 match(ConL);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immLoffset4()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4263 match(ConL);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immLoffset8()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4273 match(ConL);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immLoffset16()
4281 %{
4282 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4283 match(ConL);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 // 5 bit signed long integer
4291 operand immL5()
4292 %{
4293 predicate(Assembler::is_simm(n->get_long(), 5));
4294 match(ConL);
4295
4296 op_cost(0);
4297 format %{ %}
4298 interface(CONST_INTER);
4299 %}
4300
4301 // 7 bit unsigned long integer
4302 operand immLU7()
4303 %{
4304 predicate(Assembler::is_uimm(n->get_long(), 7));
4305 match(ConL);
4306
4307 op_cost(0);
4308 format %{ %}
4309 interface(CONST_INTER);
4310 %}
4311
4312 // 8 bit signed value.
4313 operand immI8()
4314 %{
4315 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4316 match(ConI);
4317
4318 op_cost(0);
4319 format %{ %}
4320 interface(CONST_INTER);
4321 %}
4322
4323 // 8 bit signed value (simm8), or #simm8 LSL 8.
4324 operand immIDupV()
4325 %{
4326 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4327 match(ConI);
4328
4329 op_cost(0);
4330 format %{ %}
4331 interface(CONST_INTER);
4332 %}
4333
4334 // 8 bit signed value (simm8), or #simm8 LSL 8.
4335 operand immLDupV()
4336 %{
4337 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4338 match(ConL);
4339
4340 op_cost(0);
4341 format %{ %}
4342 interface(CONST_INTER);
4343 %}
4344
4345 // 8 bit signed value (simm8), or #simm8 LSL 8.
4346 operand immHDupV()
4347 %{
4348 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4349 match(ConH);
4350
4351 op_cost(0);
4352 format %{ %}
4353 interface(CONST_INTER);
4354 %}
4355
4356 // 8 bit integer valid for vector add sub immediate
4357 operand immBAddSubV()
4358 %{
4359 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4360 match(ConI);
4361
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 // 32 bit integer valid for add sub immediate
4368 operand immIAddSub()
4369 %{
4370 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4371 match(ConI);
4372 op_cost(0);
4373 format %{ %}
4374 interface(CONST_INTER);
4375 %}
4376
4377 // 32 bit integer valid for vector add sub immediate
4378 operand immIAddSubV()
4379 %{
4380 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4381 match(ConI);
4382
4383 op_cost(0);
4384 format %{ %}
4385 interface(CONST_INTER);
4386 %}
4387
4388 // 32 bit unsigned integer valid for logical immediate
4389
4390 operand immBLog()
4391 %{
4392 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4393 match(ConI);
4394
4395 op_cost(0);
4396 format %{ %}
4397 interface(CONST_INTER);
4398 %}
4399
4400 operand immSLog()
4401 %{
4402 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4403 match(ConI);
4404
4405 op_cost(0);
4406 format %{ %}
4407 interface(CONST_INTER);
4408 %}
4409
4410 operand immILog()
4411 %{
4412 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4413 match(ConI);
4414
4415 op_cost(0);
4416 format %{ %}
4417 interface(CONST_INTER);
4418 %}
4419
4420 // Integer operands 64 bit
4421 // 64 bit immediate
4422 operand immL()
4423 %{
4424 match(ConL);
4425
4426 op_cost(0);
4427 format %{ %}
4428 interface(CONST_INTER);
4429 %}
4430
4431 // 64 bit zero
4432 operand immL0()
4433 %{
4434 predicate(n->get_long() == 0);
4435 match(ConL);
4436
4437 op_cost(0);
4438 format %{ %}
4439 interface(CONST_INTER);
4440 %}
4441
4442 // 64 bit unit decrement
4443 operand immL_M1()
4444 %{
4445 predicate(n->get_long() == -1);
4446 match(ConL);
4447
4448 op_cost(0);
4449 format %{ %}
4450 interface(CONST_INTER);
4451 %}
4452
4453 // 64 bit integer valid for add sub immediate
4454 operand immLAddSub()
4455 %{
4456 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4457 match(ConL);
4458 op_cost(0);
4459 format %{ %}
4460 interface(CONST_INTER);
4461 %}
4462
4463 // 64 bit integer valid for addv subv immediate
4464 operand immLAddSubV()
4465 %{
4466 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4467 match(ConL);
4468
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // 64 bit integer valid for logical immediate
4475 operand immLLog()
4476 %{
4477 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4478 match(ConL);
4479 op_cost(0);
4480 format %{ %}
4481 interface(CONST_INTER);
4482 %}
4483
4484 // Long Immediate: low 32-bit mask
4485 operand immL_32bits()
4486 %{
4487 predicate(n->get_long() == 0xFFFFFFFFL);
4488 match(ConL);
4489 op_cost(0);
4490 format %{ %}
4491 interface(CONST_INTER);
4492 %}
4493
4494 // Pointer operands
4495 // Pointer Immediate
4496 operand immP()
4497 %{
4498 match(ConP);
4499
4500 op_cost(0);
4501 format %{ %}
4502 interface(CONST_INTER);
4503 %}
4504
4505 // nullptr Pointer Immediate
4506 operand immP0()
4507 %{
4508 predicate(n->get_ptr() == 0);
4509 match(ConP);
4510
4511 op_cost(0);
4512 format %{ %}
4513 interface(CONST_INTER);
4514 %}
4515
4516 // Pointer Immediate One
4517 // this is used in object initialization (initial object header)
4518 operand immP_1()
4519 %{
4520 predicate(n->get_ptr() == 1);
4521 match(ConP);
4522
4523 op_cost(0);
4524 format %{ %}
4525 interface(CONST_INTER);
4526 %}
4527
4528 // AOT Runtime Constants Address
4529 operand immAOTRuntimeConstantsAddress()
4530 %{
4531 // Check if the address is in the range of AOT Runtime Constants
4532 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4533 match(ConP);
4534
4535 op_cost(0);
4536 format %{ %}
4537 interface(CONST_INTER);
4538 %}
4539
4540 // Float and Double operands
4541 // Double Immediate
4542 operand immD()
4543 %{
4544 match(ConD);
4545 op_cost(0);
4546 format %{ %}
4547 interface(CONST_INTER);
4548 %}
4549
4550 // Double Immediate: +0.0d
4551 operand immD0()
4552 %{
4553 predicate(jlong_cast(n->getd()) == 0);
4554 match(ConD);
4555
4556 op_cost(0);
4557 format %{ %}
4558 interface(CONST_INTER);
4559 %}
4560
4561 // constant 'double +0.0'.
4562 operand immDPacked()
4563 %{
4564 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4565 match(ConD);
4566 op_cost(0);
4567 format %{ %}
4568 interface(CONST_INTER);
4569 %}
4570
4571 // Float Immediate
4572 operand immF()
4573 %{
4574 match(ConF);
4575 op_cost(0);
4576 format %{ %}
4577 interface(CONST_INTER);
4578 %}
4579
4580 // Float Immediate: +0.0f.
4581 operand immF0()
4582 %{
4583 predicate(jint_cast(n->getf()) == 0);
4584 match(ConF);
4585
4586 op_cost(0);
4587 format %{ %}
4588 interface(CONST_INTER);
4589 %}
4590
4591 // Half Float (FP16) Immediate
4592 operand immH()
4593 %{
4594 match(ConH);
4595 op_cost(0);
4596 format %{ %}
4597 interface(CONST_INTER);
4598 %}
4599
4600 //
4601 operand immFPacked()
4602 %{
4603 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4604 match(ConF);
4605 op_cost(0);
4606 format %{ %}
4607 interface(CONST_INTER);
4608 %}
4609
4610 // Narrow pointer operands
4611 // Narrow Pointer Immediate
4612 operand immN()
4613 %{
4614 match(ConN);
4615
4616 op_cost(0);
4617 format %{ %}
4618 interface(CONST_INTER);
4619 %}
4620
4621 // Narrow nullptr Pointer Immediate
4622 operand immN0()
4623 %{
4624 predicate(n->get_narrowcon() == 0);
4625 match(ConN);
4626
4627 op_cost(0);
4628 format %{ %}
4629 interface(CONST_INTER);
4630 %}
4631
4632 operand immNKlass()
4633 %{
4634 match(ConNKlass);
4635
4636 op_cost(0);
4637 format %{ %}
4638 interface(CONST_INTER);
4639 %}
4640
4641 // Integer 32 bit Register Operands
4642 // Integer 32 bitRegister (excludes SP)
4643 operand iRegI()
4644 %{
4645 constraint(ALLOC_IN_RC(any_reg32));
4646 match(RegI);
4647 match(iRegINoSp);
4648 op_cost(0);
4649 format %{ %}
4650 interface(REG_INTER);
4651 %}
4652
4653 // Integer 32 bit Register not Special
4654 operand iRegINoSp()
4655 %{
4656 constraint(ALLOC_IN_RC(no_special_reg32));
4657 match(RegI);
4658 op_cost(0);
4659 format %{ %}
4660 interface(REG_INTER);
4661 %}
4662
4663 // Integer 64 bit Register Operands
4664 // Integer 64 bit Register (includes SP)
4665 operand iRegL()
4666 %{
4667 constraint(ALLOC_IN_RC(any_reg));
4668 match(RegL);
4669 match(iRegLNoSp);
4670 op_cost(0);
4671 format %{ %}
4672 interface(REG_INTER);
4673 %}
4674
4675 // Integer 64 bit Register not Special
4676 operand iRegLNoSp()
4677 %{
4678 constraint(ALLOC_IN_RC(no_special_reg));
4679 match(RegL);
4680 match(iRegL_R0);
4681 format %{ %}
4682 interface(REG_INTER);
4683 %}
4684
4685 // Pointer Register Operands
4686 // Pointer Register
4687 operand iRegP()
4688 %{
4689 constraint(ALLOC_IN_RC(ptr_reg));
4690 match(RegP);
4691 match(iRegPNoSp);
4692 match(iRegP_R0);
4693 //match(iRegP_R2);
4694 //match(iRegP_R4);
4695 match(iRegP_R5);
4696 match(thread_RegP);
4697 op_cost(0);
4698 format %{ %}
4699 interface(REG_INTER);
4700 %}
4701
4702 // Pointer 64 bit Register not Special
4703 operand iRegPNoSp()
4704 %{
4705 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4706 match(RegP);
4707 // match(iRegP);
4708 // match(iRegP_R0);
4709 // match(iRegP_R2);
4710 // match(iRegP_R4);
4711 // match(iRegP_R5);
4712 // match(thread_RegP);
4713 op_cost(0);
4714 format %{ %}
4715 interface(REG_INTER);
4716 %}
4717
4718 // This operand is not allowed to use rfp even if
4719 // rfp is not used to hold the frame pointer.
4720 operand iRegPNoSpNoRfp()
4721 %{
4722 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4723 match(RegP);
4724 match(iRegPNoSp);
4725 op_cost(0);
4726 format %{ %}
4727 interface(REG_INTER);
4728 %}
4729
4730 // Pointer 64 bit Register R0 only
4731 operand iRegP_R0()
4732 %{
4733 constraint(ALLOC_IN_RC(r0_reg));
4734 match(RegP);
4735 // match(iRegP);
4736 match(iRegPNoSp);
4737 op_cost(0);
4738 format %{ %}
4739 interface(REG_INTER);
4740 %}
4741
4742 // Pointer 64 bit Register R1 only
4743 operand iRegP_R1()
4744 %{
4745 constraint(ALLOC_IN_RC(r1_reg));
4746 match(RegP);
4747 // match(iRegP);
4748 match(iRegPNoSp);
4749 op_cost(0);
4750 format %{ %}
4751 interface(REG_INTER);
4752 %}
4753
4754 // Pointer 64 bit Register R2 only
4755 operand iRegP_R2()
4756 %{
4757 constraint(ALLOC_IN_RC(r2_reg));
4758 match(RegP);
4759 // match(iRegP);
4760 match(iRegPNoSp);
4761 op_cost(0);
4762 format %{ %}
4763 interface(REG_INTER);
4764 %}
4765
4766 // Pointer 64 bit Register R3 only
4767 operand iRegP_R3()
4768 %{
4769 constraint(ALLOC_IN_RC(r3_reg));
4770 match(RegP);
4771 // match(iRegP);
4772 match(iRegPNoSp);
4773 op_cost(0);
4774 format %{ %}
4775 interface(REG_INTER);
4776 %}
4777
4778 // Pointer 64 bit Register R4 only
4779 operand iRegP_R4()
4780 %{
4781 constraint(ALLOC_IN_RC(r4_reg));
4782 match(RegP);
4783 // match(iRegP);
4784 match(iRegPNoSp);
4785 op_cost(0);
4786 format %{ %}
4787 interface(REG_INTER);
4788 %}
4789
4790 // Pointer 64 bit Register R5 only
4791 operand iRegP_R5()
4792 %{
4793 constraint(ALLOC_IN_RC(r5_reg));
4794 match(RegP);
4795 // match(iRegP);
4796 match(iRegPNoSp);
4797 op_cost(0);
4798 format %{ %}
4799 interface(REG_INTER);
4800 %}
4801
4802 // Pointer 64 bit Register R10 only
4803 operand iRegP_R10()
4804 %{
4805 constraint(ALLOC_IN_RC(r10_reg));
4806 match(RegP);
4807 // match(iRegP);
4808 match(iRegPNoSp);
4809 op_cost(0);
4810 format %{ %}
4811 interface(REG_INTER);
4812 %}
4813
4814 // Long 64 bit Register R0 only
4815 operand iRegL_R0()
4816 %{
4817 constraint(ALLOC_IN_RC(r0_reg));
4818 match(RegL);
4819 match(iRegLNoSp);
4820 op_cost(0);
4821 format %{ %}
4822 interface(REG_INTER);
4823 %}
4824
4825 // Long 64 bit Register R11 only
4826 operand iRegL_R11()
4827 %{
4828 constraint(ALLOC_IN_RC(r11_reg));
4829 match(RegL);
4830 match(iRegLNoSp);
4831 op_cost(0);
4832 format %{ %}
4833 interface(REG_INTER);
4834 %}
4835
4836 // Register R0 only
4837 operand iRegI_R0()
4838 %{
4839 constraint(ALLOC_IN_RC(int_r0_reg));
4840 match(RegI);
4841 match(iRegINoSp);
4842 op_cost(0);
4843 format %{ %}
4844 interface(REG_INTER);
4845 %}
4846
4847 // Register R2 only
4848 operand iRegI_R2()
4849 %{
4850 constraint(ALLOC_IN_RC(int_r2_reg));
4851 match(RegI);
4852 match(iRegINoSp);
4853 op_cost(0);
4854 format %{ %}
4855 interface(REG_INTER);
4856 %}
4857
4858 // Register R3 only
4859 operand iRegI_R3()
4860 %{
4861 constraint(ALLOC_IN_RC(int_r3_reg));
4862 match(RegI);
4863 match(iRegINoSp);
4864 op_cost(0);
4865 format %{ %}
4866 interface(REG_INTER);
4867 %}
4868
4869
4870 // Register R4 only
4871 operand iRegI_R4()
4872 %{
4873 constraint(ALLOC_IN_RC(int_r4_reg));
4874 match(RegI);
4875 match(iRegINoSp);
4876 op_cost(0);
4877 format %{ %}
4878 interface(REG_INTER);
4879 %}
4880
4881
4882 // Pointer Register Operands
4883 // Narrow Pointer Register
4884 operand iRegN()
4885 %{
4886 constraint(ALLOC_IN_RC(any_reg32));
4887 match(RegN);
4888 match(iRegNNoSp);
4889 op_cost(0);
4890 format %{ %}
4891 interface(REG_INTER);
4892 %}
4893
4894 // Integer 64 bit Register not Special
4895 operand iRegNNoSp()
4896 %{
4897 constraint(ALLOC_IN_RC(no_special_reg32));
4898 match(RegN);
4899 op_cost(0);
4900 format %{ %}
4901 interface(REG_INTER);
4902 %}
4903
4904 // Float Register
4905 // Float register operands
4906 operand vRegF()
4907 %{
4908 constraint(ALLOC_IN_RC(float_reg));
4909 match(RegF);
4910
4911 op_cost(0);
4912 format %{ %}
4913 interface(REG_INTER);
4914 %}
4915
4916 // Double Register
4917 // Double register operands
4918 operand vRegD()
4919 %{
4920 constraint(ALLOC_IN_RC(double_reg));
4921 match(RegD);
4922
4923 op_cost(0);
4924 format %{ %}
4925 interface(REG_INTER);
4926 %}
4927
4928 // Generic vector class. This will be used for
4929 // all vector operands, including NEON and SVE.
4930 operand vReg()
4931 %{
4932 constraint(ALLOC_IN_RC(dynamic));
4933 match(VecA);
4934 match(VecD);
4935 match(VecX);
4936
4937 op_cost(0);
4938 format %{ %}
4939 interface(REG_INTER);
4940 %}
4941
4942 operand vReg_V10()
4943 %{
4944 constraint(ALLOC_IN_RC(v10_veca_reg));
4945 match(vReg);
4946
4947 op_cost(0);
4948 format %{ %}
4949 interface(REG_INTER);
4950 %}
4951
4952 operand vReg_V11()
4953 %{
4954 constraint(ALLOC_IN_RC(v11_veca_reg));
4955 match(vReg);
4956
4957 op_cost(0);
4958 format %{ %}
4959 interface(REG_INTER);
4960 %}
4961
4962 operand vReg_V12()
4963 %{
4964 constraint(ALLOC_IN_RC(v12_veca_reg));
4965 match(vReg);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 operand vReg_V13()
4973 %{
4974 constraint(ALLOC_IN_RC(v13_veca_reg));
4975 match(vReg);
4976
4977 op_cost(0);
4978 format %{ %}
4979 interface(REG_INTER);
4980 %}
4981
4982 operand vReg_V17()
4983 %{
4984 constraint(ALLOC_IN_RC(v17_veca_reg));
4985 match(vReg);
4986
4987 op_cost(0);
4988 format %{ %}
4989 interface(REG_INTER);
4990 %}
4991
4992 operand vReg_V18()
4993 %{
4994 constraint(ALLOC_IN_RC(v18_veca_reg));
4995 match(vReg);
4996
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 operand vReg_V23()
5003 %{
5004 constraint(ALLOC_IN_RC(v23_veca_reg));
5005 match(vReg);
5006
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vReg_V24()
5013 %{
5014 constraint(ALLOC_IN_RC(v24_veca_reg));
5015 match(vReg);
5016
5017 op_cost(0);
5018 format %{ %}
5019 interface(REG_INTER);
5020 %}
5021
5022 operand vecA()
5023 %{
5024 constraint(ALLOC_IN_RC(vectora_reg));
5025 match(VecA);
5026
5027 op_cost(0);
5028 format %{ %}
5029 interface(REG_INTER);
5030 %}
5031
5032 operand vecD()
5033 %{
5034 constraint(ALLOC_IN_RC(vectord_reg));
5035 match(VecD);
5036
5037 op_cost(0);
5038 format %{ %}
5039 interface(REG_INTER);
5040 %}
5041
5042 operand vecX()
5043 %{
5044 constraint(ALLOC_IN_RC(vectorx_reg));
5045 match(VecX);
5046
5047 op_cost(0);
5048 format %{ %}
5049 interface(REG_INTER);
5050 %}
5051
5052 operand vRegD_V0()
5053 %{
5054 constraint(ALLOC_IN_RC(v0_reg));
5055 match(RegD);
5056 op_cost(0);
5057 format %{ %}
5058 interface(REG_INTER);
5059 %}
5060
5061 operand vRegD_V1()
5062 %{
5063 constraint(ALLOC_IN_RC(v1_reg));
5064 match(RegD);
5065 op_cost(0);
5066 format %{ %}
5067 interface(REG_INTER);
5068 %}
5069
5070 operand vRegD_V2()
5071 %{
5072 constraint(ALLOC_IN_RC(v2_reg));
5073 match(RegD);
5074 op_cost(0);
5075 format %{ %}
5076 interface(REG_INTER);
5077 %}
5078
5079 operand vRegD_V3()
5080 %{
5081 constraint(ALLOC_IN_RC(v3_reg));
5082 match(RegD);
5083 op_cost(0);
5084 format %{ %}
5085 interface(REG_INTER);
5086 %}
5087
5088 operand vRegD_V4()
5089 %{
5090 constraint(ALLOC_IN_RC(v4_reg));
5091 match(RegD);
5092 op_cost(0);
5093 format %{ %}
5094 interface(REG_INTER);
5095 %}
5096
5097 operand vRegD_V5()
5098 %{
5099 constraint(ALLOC_IN_RC(v5_reg));
5100 match(RegD);
5101 op_cost(0);
5102 format %{ %}
5103 interface(REG_INTER);
5104 %}
5105
5106 operand vRegD_V6()
5107 %{
5108 constraint(ALLOC_IN_RC(v6_reg));
5109 match(RegD);
5110 op_cost(0);
5111 format %{ %}
5112 interface(REG_INTER);
5113 %}
5114
5115 operand vRegD_V7()
5116 %{
5117 constraint(ALLOC_IN_RC(v7_reg));
5118 match(RegD);
5119 op_cost(0);
5120 format %{ %}
5121 interface(REG_INTER);
5122 %}
5123
5124 operand vRegD_V12()
5125 %{
5126 constraint(ALLOC_IN_RC(v12_reg));
5127 match(RegD);
5128 op_cost(0);
5129 format %{ %}
5130 interface(REG_INTER);
5131 %}
5132
5133 operand vRegD_V13()
5134 %{
5135 constraint(ALLOC_IN_RC(v13_reg));
5136 match(RegD);
5137 op_cost(0);
5138 format %{ %}
5139 interface(REG_INTER);
5140 %}
5141
5142 operand pReg()
5143 %{
5144 constraint(ALLOC_IN_RC(pr_reg));
5145 match(RegVectMask);
5146 match(pRegGov);
5147 op_cost(0);
5148 format %{ %}
5149 interface(REG_INTER);
5150 %}
5151
5152 operand pRegGov()
5153 %{
5154 constraint(ALLOC_IN_RC(gov_pr));
5155 match(RegVectMask);
5156 match(pReg);
5157 op_cost(0);
5158 format %{ %}
5159 interface(REG_INTER);
5160 %}
5161
5162 operand pRegGov_P0()
5163 %{
5164 constraint(ALLOC_IN_RC(p0_reg));
5165 match(RegVectMask);
5166 op_cost(0);
5167 format %{ %}
5168 interface(REG_INTER);
5169 %}
5170
5171 operand pRegGov_P1()
5172 %{
5173 constraint(ALLOC_IN_RC(p1_reg));
5174 match(RegVectMask);
5175 op_cost(0);
5176 format %{ %}
5177 interface(REG_INTER);
5178 %}
5179
5180 // Flags register, used as output of signed compare instructions
5181
5182 // note that on AArch64 we also use this register as the output for
5183 // for floating point compare instructions (CmpF CmpD). this ensures
5184 // that ordered inequality tests use GT, GE, LT or LE none of which
5185 // pass through cases where the result is unordered i.e. one or both
5186 // inputs to the compare is a NaN. this means that the ideal code can
5187 // replace e.g. a GT with an LE and not end up capturing the NaN case
5188 // (where the comparison should always fail). EQ and NE tests are
5189 // always generated in ideal code so that unordered folds into the NE
5190 // case, matching the behaviour of AArch64 NE.
5191 //
5192 // This differs from x86 where the outputs of FP compares use a
5193 // special FP flags registers and where compares based on this
5194 // register are distinguished into ordered inequalities (cmpOpUCF) and
5195 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5196 // to explicitly handle the unordered case in branches. x86 also has
5197 // to include extra CMoveX rules to accept a cmpOpUCF input.
5198
5199 operand rFlagsReg()
5200 %{
5201 constraint(ALLOC_IN_RC(int_flags));
5202 match(RegFlags);
5203
5204 op_cost(0);
5205 format %{ "RFLAGS" %}
5206 interface(REG_INTER);
5207 %}
5208
5209 // Flags register, used as output of unsigned compare instructions
5210 operand rFlagsRegU()
5211 %{
5212 constraint(ALLOC_IN_RC(int_flags));
5213 match(RegFlags);
5214
5215 op_cost(0);
5216 format %{ "RFLAGSU" %}
5217 interface(REG_INTER);
5218 %}
5219
5220 // Special Registers
5221
5222 // Method Register
5223 operand inline_cache_RegP(iRegP reg)
5224 %{
5225 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5226 match(reg);
5227 match(iRegPNoSp);
5228 op_cost(0);
5229 format %{ %}
5230 interface(REG_INTER);
5231 %}
5232
5233 // Thread Register
5234 operand thread_RegP(iRegP reg)
5235 %{
5236 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5237 match(reg);
5238 op_cost(0);
5239 format %{ %}
5240 interface(REG_INTER);
5241 %}
5242
5243 //----------Memory Operands----------------------------------------------------
5244
5245 operand indirect(iRegP reg)
5246 %{
5247 constraint(ALLOC_IN_RC(ptr_reg));
5248 match(reg);
5249 op_cost(0);
5250 format %{ "[$reg]" %}
5251 interface(MEMORY_INTER) %{
5252 base($reg);
5253 index(0xffffffff);
5254 scale(0x0);
5255 disp(0x0);
5256 %}
5257 %}
5258
5259 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5260 %{
5261 constraint(ALLOC_IN_RC(ptr_reg));
5262 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5263 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5264 op_cost(0);
5265 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5266 interface(MEMORY_INTER) %{
5267 base($reg);
5268 index($ireg);
5269 scale($scale);
5270 disp(0x0);
5271 %}
5272 %}
5273
5274 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5275 %{
5276 constraint(ALLOC_IN_RC(ptr_reg));
5277 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5278 match(AddP reg (LShiftL lreg scale));
5279 op_cost(0);
5280 format %{ "$reg, $lreg lsl($scale)" %}
5281 interface(MEMORY_INTER) %{
5282 base($reg);
5283 index($lreg);
5284 scale($scale);
5285 disp(0x0);
5286 %}
5287 %}
5288
5289 operand indIndexI2L(iRegP reg, iRegI ireg)
5290 %{
5291 constraint(ALLOC_IN_RC(ptr_reg));
5292 match(AddP reg (ConvI2L ireg));
5293 op_cost(0);
5294 format %{ "$reg, $ireg, 0, I2L" %}
5295 interface(MEMORY_INTER) %{
5296 base($reg);
5297 index($ireg);
5298 scale(0x0);
5299 disp(0x0);
5300 %}
5301 %}
5302
5303 operand indIndex(iRegP reg, iRegL lreg)
5304 %{
5305 constraint(ALLOC_IN_RC(ptr_reg));
5306 match(AddP reg lreg);
5307 op_cost(0);
5308 format %{ "$reg, $lreg" %}
5309 interface(MEMORY_INTER) %{
5310 base($reg);
5311 index($lreg);
5312 scale(0x0);
5313 disp(0x0);
5314 %}
5315 %}
5316
5317 operand indOffI1(iRegP reg, immIOffset1 off)
5318 %{
5319 constraint(ALLOC_IN_RC(ptr_reg));
5320 match(AddP reg off);
5321 op_cost(0);
5322 format %{ "[$reg, $off]" %}
5323 interface(MEMORY_INTER) %{
5324 base($reg);
5325 index(0xffffffff);
5326 scale(0x0);
5327 disp($off);
5328 %}
5329 %}
5330
5331 operand indOffI2(iRegP reg, immIOffset2 off)
5332 %{
5333 constraint(ALLOC_IN_RC(ptr_reg));
5334 match(AddP reg off);
5335 op_cost(0);
5336 format %{ "[$reg, $off]" %}
5337 interface(MEMORY_INTER) %{
5338 base($reg);
5339 index(0xffffffff);
5340 scale(0x0);
5341 disp($off);
5342 %}
5343 %}
5344
5345 operand indOffI4(iRegP reg, immIOffset4 off)
5346 %{
5347 constraint(ALLOC_IN_RC(ptr_reg));
5348 match(AddP reg off);
5349 op_cost(0);
5350 format %{ "[$reg, $off]" %}
5351 interface(MEMORY_INTER) %{
5352 base($reg);
5353 index(0xffffffff);
5354 scale(0x0);
5355 disp($off);
5356 %}
5357 %}
5358
5359 operand indOffI8(iRegP reg, immIOffset8 off)
5360 %{
5361 constraint(ALLOC_IN_RC(ptr_reg));
5362 match(AddP reg off);
5363 op_cost(0);
5364 format %{ "[$reg, $off]" %}
5365 interface(MEMORY_INTER) %{
5366 base($reg);
5367 index(0xffffffff);
5368 scale(0x0);
5369 disp($off);
5370 %}
5371 %}
5372
5373 operand indOffI16(iRegP reg, immIOffset16 off)
5374 %{
5375 constraint(ALLOC_IN_RC(ptr_reg));
5376 match(AddP reg off);
5377 op_cost(0);
5378 format %{ "[$reg, $off]" %}
5379 interface(MEMORY_INTER) %{
5380 base($reg);
5381 index(0xffffffff);
5382 scale(0x0);
5383 disp($off);
5384 %}
5385 %}
5386
5387 operand indOffL1(iRegP reg, immLoffset1 off)
5388 %{
5389 constraint(ALLOC_IN_RC(ptr_reg));
5390 match(AddP reg off);
5391 op_cost(0);
5392 format %{ "[$reg, $off]" %}
5393 interface(MEMORY_INTER) %{
5394 base($reg);
5395 index(0xffffffff);
5396 scale(0x0);
5397 disp($off);
5398 %}
5399 %}
5400
5401 operand indOffL2(iRegP reg, immLoffset2 off)
5402 %{
5403 constraint(ALLOC_IN_RC(ptr_reg));
5404 match(AddP reg off);
5405 op_cost(0);
5406 format %{ "[$reg, $off]" %}
5407 interface(MEMORY_INTER) %{
5408 base($reg);
5409 index(0xffffffff);
5410 scale(0x0);
5411 disp($off);
5412 %}
5413 %}
5414
5415 operand indOffL4(iRegP reg, immLoffset4 off)
5416 %{
5417 constraint(ALLOC_IN_RC(ptr_reg));
5418 match(AddP reg off);
5419 op_cost(0);
5420 format %{ "[$reg, $off]" %}
5421 interface(MEMORY_INTER) %{
5422 base($reg);
5423 index(0xffffffff);
5424 scale(0x0);
5425 disp($off);
5426 %}
5427 %}
5428
5429 operand indOffL8(iRegP reg, immLoffset8 off)
5430 %{
5431 constraint(ALLOC_IN_RC(ptr_reg));
5432 match(AddP reg off);
5433 op_cost(0);
5434 format %{ "[$reg, $off]" %}
5435 interface(MEMORY_INTER) %{
5436 base($reg);
5437 index(0xffffffff);
5438 scale(0x0);
5439 disp($off);
5440 %}
5441 %}
5442
5443 operand indOffL16(iRegP reg, immLoffset16 off)
5444 %{
5445 constraint(ALLOC_IN_RC(ptr_reg));
5446 match(AddP reg off);
5447 op_cost(0);
5448 format %{ "[$reg, $off]" %}
5449 interface(MEMORY_INTER) %{
5450 base($reg);
5451 index(0xffffffff);
5452 scale(0x0);
5453 disp($off);
5454 %}
5455 %}
5456
5457 operand indirectX2P(iRegL reg)
5458 %{
5459 constraint(ALLOC_IN_RC(ptr_reg));
5460 match(CastX2P reg);
5461 op_cost(0);
5462 format %{ "[$reg]\t# long -> ptr" %}
5463 interface(MEMORY_INTER) %{
5464 base($reg);
5465 index(0xffffffff);
5466 scale(0x0);
5467 disp(0x0);
5468 %}
5469 %}
5470
5471 operand indOffX2P(iRegL reg, immLOffset off)
5472 %{
5473 constraint(ALLOC_IN_RC(ptr_reg));
5474 match(AddP (CastX2P reg) off);
5475 op_cost(0);
5476 format %{ "[$reg, $off]\t# long -> ptr" %}
5477 interface(MEMORY_INTER) %{
5478 base($reg);
5479 index(0xffffffff);
5480 scale(0x0);
5481 disp($off);
5482 %}
5483 %}
5484
5485 operand indirectN(iRegN reg)
5486 %{
5487 predicate(CompressedOops::shift() == 0);
5488 constraint(ALLOC_IN_RC(ptr_reg));
5489 match(DecodeN reg);
5490 op_cost(0);
5491 format %{ "[$reg]\t# narrow" %}
5492 interface(MEMORY_INTER) %{
5493 base($reg);
5494 index(0xffffffff);
5495 scale(0x0);
5496 disp(0x0);
5497 %}
5498 %}
5499
5500 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5501 %{
5502 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5503 constraint(ALLOC_IN_RC(ptr_reg));
5504 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5505 op_cost(0);
5506 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5507 interface(MEMORY_INTER) %{
5508 base($reg);
5509 index($ireg);
5510 scale($scale);
5511 disp(0x0);
5512 %}
5513 %}
5514
5515 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5516 %{
5517 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5518 constraint(ALLOC_IN_RC(ptr_reg));
5519 match(AddP (DecodeN reg) (LShiftL lreg scale));
5520 op_cost(0);
5521 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5522 interface(MEMORY_INTER) %{
5523 base($reg);
5524 index($lreg);
5525 scale($scale);
5526 disp(0x0);
5527 %}
5528 %}
5529
5530 operand indIndexI2LN(iRegN reg, iRegI ireg)
5531 %{
5532 predicate(CompressedOops::shift() == 0);
5533 constraint(ALLOC_IN_RC(ptr_reg));
5534 match(AddP (DecodeN reg) (ConvI2L ireg));
5535 op_cost(0);
5536 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5537 interface(MEMORY_INTER) %{
5538 base($reg);
5539 index($ireg);
5540 scale(0x0);
5541 disp(0x0);
5542 %}
5543 %}
5544
5545 operand indIndexN(iRegN reg, iRegL lreg)
5546 %{
5547 predicate(CompressedOops::shift() == 0);
5548 constraint(ALLOC_IN_RC(ptr_reg));
5549 match(AddP (DecodeN reg) lreg);
5550 op_cost(0);
5551 format %{ "$reg, $lreg\t# narrow" %}
5552 interface(MEMORY_INTER) %{
5553 base($reg);
5554 index($lreg);
5555 scale(0x0);
5556 disp(0x0);
5557 %}
5558 %}
5559
5560 operand indOffIN(iRegN reg, immIOffset off)
5561 %{
5562 predicate(CompressedOops::shift() == 0);
5563 constraint(ALLOC_IN_RC(ptr_reg));
5564 match(AddP (DecodeN reg) off);
5565 op_cost(0);
5566 format %{ "[$reg, $off]\t# narrow" %}
5567 interface(MEMORY_INTER) %{
5568 base($reg);
5569 index(0xffffffff);
5570 scale(0x0);
5571 disp($off);
5572 %}
5573 %}
5574
5575 operand indOffLN(iRegN reg, immLOffset off)
5576 %{
5577 predicate(CompressedOops::shift() == 0);
5578 constraint(ALLOC_IN_RC(ptr_reg));
5579 match(AddP (DecodeN reg) off);
5580 op_cost(0);
5581 format %{ "[$reg, $off]\t# narrow" %}
5582 interface(MEMORY_INTER) %{
5583 base($reg);
5584 index(0xffffffff);
5585 scale(0x0);
5586 disp($off);
5587 %}
5588 %}
5589
5590
5591 //----------Special Memory Operands--------------------------------------------
5592 // Stack Slot Operand - This operand is used for loading and storing temporary
5593 // values on the stack where a match requires a value to
5594 // flow through memory.
5595 operand stackSlotP(sRegP reg)
5596 %{
5597 constraint(ALLOC_IN_RC(stack_slots));
5598 op_cost(100);
5599 // No match rule because this operand is only generated in matching
5600 // match(RegP);
5601 format %{ "[$reg]" %}
5602 interface(MEMORY_INTER) %{
5603 base(0x1e); // RSP
5604 index(0x0); // No Index
5605 scale(0x0); // No Scale
5606 disp($reg); // Stack Offset
5607 %}
5608 %}
5609
5610 operand stackSlotI(sRegI reg)
5611 %{
5612 constraint(ALLOC_IN_RC(stack_slots));
5613 // No match rule because this operand is only generated in matching
5614 // match(RegI);
5615 format %{ "[$reg]" %}
5616 interface(MEMORY_INTER) %{
5617 base(0x1e); // RSP
5618 index(0x0); // No Index
5619 scale(0x0); // No Scale
5620 disp($reg); // Stack Offset
5621 %}
5622 %}
5623
5624 operand stackSlotF(sRegF reg)
5625 %{
5626 constraint(ALLOC_IN_RC(stack_slots));
5627 // No match rule because this operand is only generated in matching
5628 // match(RegF);
5629 format %{ "[$reg]" %}
5630 interface(MEMORY_INTER) %{
5631 base(0x1e); // RSP
5632 index(0x0); // No Index
5633 scale(0x0); // No Scale
5634 disp($reg); // Stack Offset
5635 %}
5636 %}
5637
5638 operand stackSlotD(sRegD reg)
5639 %{
5640 constraint(ALLOC_IN_RC(stack_slots));
5641 // No match rule because this operand is only generated in matching
5642 // match(RegD);
5643 format %{ "[$reg]" %}
5644 interface(MEMORY_INTER) %{
5645 base(0x1e); // RSP
5646 index(0x0); // No Index
5647 scale(0x0); // No Scale
5648 disp($reg); // Stack Offset
5649 %}
5650 %}
5651
5652 operand stackSlotL(sRegL reg)
5653 %{
5654 constraint(ALLOC_IN_RC(stack_slots));
5655 // No match rule because this operand is only generated in matching
5656 // match(RegL);
5657 format %{ "[$reg]" %}
5658 interface(MEMORY_INTER) %{
5659 base(0x1e); // RSP
5660 index(0x0); // No Index
5661 scale(0x0); // No Scale
5662 disp($reg); // Stack Offset
5663 %}
5664 %}
5665
5666 // Operands for expressing Control Flow
5667 // NOTE: Label is a predefined operand which should not be redefined in
5668 // the AD file. It is generically handled within the ADLC.
5669
5670 //----------Conditional Branch Operands----------------------------------------
5671 // Comparison Op - This is the operation of the comparison, and is limited to
5672 // the following set of codes:
5673 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5674 //
5675 // Other attributes of the comparison, such as unsignedness, are specified
5676 // by the comparison instruction that sets a condition code flags register.
5677 // That result is represented by a flags operand whose subtype is appropriate
5678 // to the unsignedness (etc.) of the comparison.
5679 //
5680 // Later, the instruction which matches both the Comparison Op (a Bool) and
5681 // the flags (produced by the Cmp) specifies the coding of the comparison op
5682 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5683
5684 // used for signed integral comparisons and fp comparisons
5685
5686 operand cmpOp()
5687 %{
5688 match(Bool);
5689
5690 format %{ "" %}
5691 interface(COND_INTER) %{
5692 equal(0x0, "eq");
5693 not_equal(0x1, "ne");
5694 less(0xb, "lt");
5695 greater_equal(0xa, "ge");
5696 less_equal(0xd, "le");
5697 greater(0xc, "gt");
5698 overflow(0x6, "vs");
5699 no_overflow(0x7, "vc");
5700 %}
5701 %}
5702
5703 // used for unsigned integral comparisons
5704
5705 operand cmpOpU()
5706 %{
5707 match(Bool);
5708
5709 format %{ "" %}
5710 interface(COND_INTER) %{
5711 equal(0x0, "eq");
5712 not_equal(0x1, "ne");
5713 less(0x3, "lo");
5714 greater_equal(0x2, "hs");
5715 less_equal(0x9, "ls");
5716 greater(0x8, "hi");
5717 overflow(0x6, "vs");
5718 no_overflow(0x7, "vc");
5719 %}
5720 %}
5721
5722 // used for certain integral comparisons which can be
5723 // converted to cbxx or tbxx instructions
5724
5725 operand cmpOpEqNe()
5726 %{
5727 match(Bool);
5728 op_cost(0);
5729 predicate(n->as_Bool()->_test._test == BoolTest::ne
5730 || n->as_Bool()->_test._test == BoolTest::eq);
5731
5732 format %{ "" %}
5733 interface(COND_INTER) %{
5734 equal(0x0, "eq");
5735 not_equal(0x1, "ne");
5736 less(0xb, "lt");
5737 greater_equal(0xa, "ge");
5738 less_equal(0xd, "le");
5739 greater(0xc, "gt");
5740 overflow(0x6, "vs");
5741 no_overflow(0x7, "vc");
5742 %}
5743 %}
5744
5745 // used for certain integral comparisons which can be
5746 // converted to cbxx or tbxx instructions
5747
5748 operand cmpOpLtGe()
5749 %{
5750 match(Bool);
5751 op_cost(0);
5752
5753 predicate(n->as_Bool()->_test._test == BoolTest::lt
5754 || n->as_Bool()->_test._test == BoolTest::ge);
5755
5756 format %{ "" %}
5757 interface(COND_INTER) %{
5758 equal(0x0, "eq");
5759 not_equal(0x1, "ne");
5760 less(0xb, "lt");
5761 greater_equal(0xa, "ge");
5762 less_equal(0xd, "le");
5763 greater(0xc, "gt");
5764 overflow(0x6, "vs");
5765 no_overflow(0x7, "vc");
5766 %}
5767 %}
5768
5769 // used for certain unsigned integral comparisons which can be
5770 // converted to cbxx or tbxx instructions
5771
5772 operand cmpOpUEqNeLeGt()
5773 %{
5774 match(Bool);
5775 op_cost(0);
5776
5777 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5778 n->as_Bool()->_test._test == BoolTest::ne ||
5779 n->as_Bool()->_test._test == BoolTest::le ||
5780 n->as_Bool()->_test._test == BoolTest::gt);
5781
5782 format %{ "" %}
5783 interface(COND_INTER) %{
5784 equal(0x0, "eq");
5785 not_equal(0x1, "ne");
5786 less(0x3, "lo");
5787 greater_equal(0x2, "hs");
5788 less_equal(0x9, "ls");
5789 greater(0x8, "hi");
5790 overflow(0x6, "vs");
5791 no_overflow(0x7, "vc");
5792 %}
5793 %}
5794
5795 // Special operand allowing long args to int ops to be truncated for free
5796
5797 operand iRegL2I(iRegL reg) %{
5798
5799 op_cost(0);
5800
5801 match(ConvL2I reg);
5802
5803 format %{ "l2i($reg)" %}
5804
5805 interface(REG_INTER)
5806 %}
5807
5808 operand iRegL2P(iRegL reg) %{
5809
5810 op_cost(0);
5811
5812 match(CastX2P reg);
5813
5814 format %{ "l2p($reg)" %}
5815
5816 interface(REG_INTER)
5817 %}
5818
5819 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5820 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5821 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5822 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5823
5824 //----------OPERAND CLASSES----------------------------------------------------
5825 // Operand Classes are groups of operands that are used as to simplify
5826 // instruction definitions by not requiring the AD writer to specify
5827 // separate instructions for every form of operand when the
5828 // instruction accepts multiple operand types with the same basic
5829 // encoding and format. The classic case of this is memory operands.
5830
5831 // memory is used to define read/write location for load/store
5832 // instruction defs. we can turn a memory op into an Address
5833
5834 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5835 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5836
5837 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5838 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5839
5840 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5841 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5842
5843 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5844 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5845
5846 // All of the memory operands. For the pipeline description.
5847 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5848 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5849 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5850
5851 opclass memory_noindex(indirect,
5852 indOffI1, indOffL1,indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5853 indirectN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5854
5855 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5856 // operations. it allows the src to be either an iRegI or a (ConvL2I
5857 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5858 // can be elided because the 32-bit instruction will just employ the
5859 // lower 32 bits anyway.
5860 //
5861 // n.b. this does not elide all L2I conversions. if the truncated
5862 // value is consumed by more than one operation then the ConvL2I
5863 // cannot be bundled into the consuming nodes so an l2i gets planted
5864 // (actually a movw $dst $src) and the downstream instructions consume
5865 // the result of the l2i as an iRegI input. That's a shame since the
5866 // movw is actually redundant but its not too costly.
5867
5868 opclass iRegIorL2I(iRegI, iRegL2I);
5869 opclass iRegPorL2P(iRegP, iRegL2P);
5870
5871 //----------PIPELINE-----------------------------------------------------------
5872 // Rules which define the behavior of the target architectures pipeline.
5873
5874 // For specific pipelines, eg A53, define the stages of that pipeline
5875 //pipe_desc(ISS, EX1, EX2, WR);
5876 #define ISS S0
5877 #define EX1 S1
5878 #define EX2 S2
5879 #define WR S3
5880
5881 // Integer ALU reg operation
5882 pipeline %{
5883
5884 attributes %{
5885 // ARM instructions are of fixed length
5886 fixed_size_instructions; // Fixed size instructions TODO does
5887 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5888 // ARM instructions come in 32-bit word units
5889 instruction_unit_size = 4; // An instruction is 4 bytes long
5890 instruction_fetch_unit_size = 64; // The processor fetches one line
5891 instruction_fetch_units = 1; // of 64 bytes
5892 %}
5893
5894 // We don't use an actual pipeline model so don't care about resources
5895 // or description. we do use pipeline classes to introduce fixed
5896 // latencies
5897
5898 //----------RESOURCES----------------------------------------------------------
5899 // Resources are the functional units available to the machine
5900
5901 resources( INS0, INS1, INS01 = INS0 | INS1,
5902 ALU0, ALU1, ALU = ALU0 | ALU1,
5903 MAC,
5904 DIV,
5905 BRANCH,
5906 LDST,
5907 NEON_FP);
5908
5909 //----------PIPELINE DESCRIPTION-----------------------------------------------
5910 // Pipeline Description specifies the stages in the machine's pipeline
5911
5912 // Define the pipeline as a generic 6 stage pipeline
5913 pipe_desc(S0, S1, S2, S3, S4, S5);
5914
5915 //----------PIPELINE CLASSES---------------------------------------------------
5916 // Pipeline Classes describe the stages in which input and output are
5917 // referenced by the hardware pipeline.
5918
5919 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5920 %{
5921 single_instruction;
5922 src1 : S1(read);
5923 src2 : S2(read);
5924 dst : S5(write);
5925 INS01 : ISS;
5926 NEON_FP : S5;
5927 %}
5928
5929 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5930 %{
5931 single_instruction;
5932 src1 : S1(read);
5933 src2 : S2(read);
5934 dst : S5(write);
5935 INS01 : ISS;
5936 NEON_FP : S5;
5937 %}
5938
5939 pipe_class fp_uop_s(vRegF dst, vRegF src)
5940 %{
5941 single_instruction;
5942 src : S1(read);
5943 dst : S5(write);
5944 INS01 : ISS;
5945 NEON_FP : S5;
5946 %}
5947
5948 pipe_class fp_uop_d(vRegD dst, vRegD src)
5949 %{
5950 single_instruction;
5951 src : S1(read);
5952 dst : S5(write);
5953 INS01 : ISS;
5954 NEON_FP : S5;
5955 %}
5956
5957 pipe_class fp_d2f(vRegF dst, vRegD src)
5958 %{
5959 single_instruction;
5960 src : S1(read);
5961 dst : S5(write);
5962 INS01 : ISS;
5963 NEON_FP : S5;
5964 %}
5965
5966 pipe_class fp_f2d(vRegD dst, vRegF src)
5967 %{
5968 single_instruction;
5969 src : S1(read);
5970 dst : S5(write);
5971 INS01 : ISS;
5972 NEON_FP : S5;
5973 %}
5974
5975 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5976 %{
5977 single_instruction;
5978 src : S1(read);
5979 dst : S5(write);
5980 INS01 : ISS;
5981 NEON_FP : S5;
5982 %}
5983
5984 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5985 %{
5986 single_instruction;
5987 src : S1(read);
5988 dst : S5(write);
5989 INS01 : ISS;
5990 NEON_FP : S5;
5991 %}
5992
5993 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5994 %{
5995 single_instruction;
5996 src : S1(read);
5997 dst : S5(write);
5998 INS01 : ISS;
5999 NEON_FP : S5;
6000 %}
6001
6002 pipe_class fp_l2f(vRegF dst, iRegL src)
6003 %{
6004 single_instruction;
6005 src : S1(read);
6006 dst : S5(write);
6007 INS01 : ISS;
6008 NEON_FP : S5;
6009 %}
6010
6011 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6012 %{
6013 single_instruction;
6014 src : S1(read);
6015 dst : S5(write);
6016 INS01 : ISS;
6017 NEON_FP : S5;
6018 %}
6019
6020 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6021 %{
6022 single_instruction;
6023 src : S1(read);
6024 dst : S5(write);
6025 INS01 : ISS;
6026 NEON_FP : S5;
6027 %}
6028
6029 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6030 %{
6031 single_instruction;
6032 src : S1(read);
6033 dst : S5(write);
6034 INS01 : ISS;
6035 NEON_FP : S5;
6036 %}
6037
6038 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6039 %{
6040 single_instruction;
6041 src : S1(read);
6042 dst : S5(write);
6043 INS01 : ISS;
6044 NEON_FP : S5;
6045 %}
6046
6047 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6048 %{
6049 single_instruction;
6050 src1 : S1(read);
6051 src2 : S2(read);
6052 dst : S5(write);
6053 INS0 : ISS;
6054 NEON_FP : S5;
6055 %}
6056
6057 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6058 %{
6059 single_instruction;
6060 src1 : S1(read);
6061 src2 : S2(read);
6062 dst : S5(write);
6063 INS0 : ISS;
6064 NEON_FP : S5;
6065 %}
6066
6067 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6068 %{
6069 single_instruction;
6070 cr : S1(read);
6071 src1 : S1(read);
6072 src2 : S1(read);
6073 dst : S3(write);
6074 INS01 : ISS;
6075 NEON_FP : S3;
6076 %}
6077
6078 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6079 %{
6080 single_instruction;
6081 cr : S1(read);
6082 src1 : S1(read);
6083 src2 : S1(read);
6084 dst : S3(write);
6085 INS01 : ISS;
6086 NEON_FP : S3;
6087 %}
6088
6089 pipe_class fp_imm_s(vRegF dst)
6090 %{
6091 single_instruction;
6092 dst : S3(write);
6093 INS01 : ISS;
6094 NEON_FP : S3;
6095 %}
6096
6097 pipe_class fp_imm_d(vRegD dst)
6098 %{
6099 single_instruction;
6100 dst : S3(write);
6101 INS01 : ISS;
6102 NEON_FP : S3;
6103 %}
6104
6105 pipe_class fp_load_constant_s(vRegF dst)
6106 %{
6107 single_instruction;
6108 dst : S4(write);
6109 INS01 : ISS;
6110 NEON_FP : S4;
6111 %}
6112
6113 pipe_class fp_load_constant_d(vRegD dst)
6114 %{
6115 single_instruction;
6116 dst : S4(write);
6117 INS01 : ISS;
6118 NEON_FP : S4;
6119 %}
6120
6121 //------- Integer ALU operations --------------------------
6122
6123 // Integer ALU reg-reg operation
6124 // Operands needed in EX1, result generated in EX2
6125 // Eg. ADD x0, x1, x2
6126 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6127 %{
6128 single_instruction;
6129 dst : EX2(write);
6130 src1 : EX1(read);
6131 src2 : EX1(read);
6132 INS01 : ISS; // Dual issue as instruction 0 or 1
6133 ALU : EX2;
6134 %}
6135
6136 // Integer ALU reg-reg operation with constant shift
6137 // Shifted register must be available in LATE_ISS instead of EX1
6138 // Eg. ADD x0, x1, x2, LSL #2
6139 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6140 %{
6141 single_instruction;
6142 dst : EX2(write);
6143 src1 : EX1(read);
6144 src2 : ISS(read);
6145 INS01 : ISS;
6146 ALU : EX2;
6147 %}
6148
6149 // Integer ALU reg operation with constant shift
6150 // Eg. LSL x0, x1, #shift
6151 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6152 %{
6153 single_instruction;
6154 dst : EX2(write);
6155 src1 : ISS(read);
6156 INS01 : ISS;
6157 ALU : EX2;
6158 %}
6159
6160 // Integer ALU reg-reg operation with variable shift
6161 // Both operands must be available in LATE_ISS instead of EX1
6162 // Result is available in EX1 instead of EX2
6163 // Eg. LSLV x0, x1, x2
6164 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6165 %{
6166 single_instruction;
6167 dst : EX1(write);
6168 src1 : ISS(read);
6169 src2 : ISS(read);
6170 INS01 : ISS;
6171 ALU : EX1;
6172 %}
6173
6174 // Integer ALU reg-reg operation with extract
6175 // As for _vshift above, but result generated in EX2
6176 // Eg. EXTR x0, x1, x2, #N
6177 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6178 %{
6179 single_instruction;
6180 dst : EX2(write);
6181 src1 : ISS(read);
6182 src2 : ISS(read);
6183 INS1 : ISS; // Can only dual issue as Instruction 1
6184 ALU : EX1;
6185 %}
6186
6187 // Integer ALU reg operation
6188 // Eg. NEG x0, x1
6189 pipe_class ialu_reg(iRegI dst, iRegI src)
6190 %{
6191 single_instruction;
6192 dst : EX2(write);
6193 src : EX1(read);
6194 INS01 : ISS;
6195 ALU : EX2;
6196 %}
6197
6198 // Integer ALU reg mmediate operation
6199 // Eg. ADD x0, x1, #N
6200 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6201 %{
6202 single_instruction;
6203 dst : EX2(write);
6204 src1 : EX1(read);
6205 INS01 : ISS;
6206 ALU : EX2;
6207 %}
6208
6209 // Integer ALU immediate operation (no source operands)
6210 // Eg. MOV x0, #N
6211 pipe_class ialu_imm(iRegI dst)
6212 %{
6213 single_instruction;
6214 dst : EX1(write);
6215 INS01 : ISS;
6216 ALU : EX1;
6217 %}
6218
6219 //------- Compare operation -------------------------------
6220
6221 // Compare reg-reg
6222 // Eg. CMP x0, x1
6223 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6224 %{
6225 single_instruction;
6226 // fixed_latency(16);
6227 cr : EX2(write);
6228 op1 : EX1(read);
6229 op2 : EX1(read);
6230 INS01 : ISS;
6231 ALU : EX2;
6232 %}
6233
6234 // Compare reg-reg
6235 // Eg. CMP x0, #N
6236 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6237 %{
6238 single_instruction;
6239 // fixed_latency(16);
6240 cr : EX2(write);
6241 op1 : EX1(read);
6242 INS01 : ISS;
6243 ALU : EX2;
6244 %}
6245
6246 //------- Conditional instructions ------------------------
6247
6248 // Conditional no operands
6249 // Eg. CSINC x0, zr, zr, <cond>
6250 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6251 %{
6252 single_instruction;
6253 cr : EX1(read);
6254 dst : EX2(write);
6255 INS01 : ISS;
6256 ALU : EX2;
6257 %}
6258
6259 // Conditional 2 operand
6260 // EG. CSEL X0, X1, X2, <cond>
6261 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6262 %{
6263 single_instruction;
6264 cr : EX1(read);
6265 src1 : EX1(read);
6266 src2 : EX1(read);
6267 dst : EX2(write);
6268 INS01 : ISS;
6269 ALU : EX2;
6270 %}
6271
6272 // Conditional 2 operand
6273 // EG. CSEL X0, X1, X2, <cond>
6274 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6275 %{
6276 single_instruction;
6277 cr : EX1(read);
6278 src : EX1(read);
6279 dst : EX2(write);
6280 INS01 : ISS;
6281 ALU : EX2;
6282 %}
6283
6284 //------- Multiply pipeline operations --------------------
6285
6286 // Multiply reg-reg
6287 // Eg. MUL w0, w1, w2
6288 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6289 %{
6290 single_instruction;
6291 dst : WR(write);
6292 src1 : ISS(read);
6293 src2 : ISS(read);
6294 INS01 : ISS;
6295 MAC : WR;
6296 %}
6297
6298 // Multiply accumulate
6299 // Eg. MADD w0, w1, w2, w3
6300 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6301 %{
6302 single_instruction;
6303 dst : WR(write);
6304 src1 : ISS(read);
6305 src2 : ISS(read);
6306 src3 : ISS(read);
6307 INS01 : ISS;
6308 MAC : WR;
6309 %}
6310
6311 // Eg. MUL w0, w1, w2
6312 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6313 %{
6314 single_instruction;
6315 fixed_latency(3); // Maximum latency for 64 bit mul
6316 dst : WR(write);
6317 src1 : ISS(read);
6318 src2 : ISS(read);
6319 INS01 : ISS;
6320 MAC : WR;
6321 %}
6322
6323 // Multiply accumulate
6324 // Eg. MADD w0, w1, w2, w3
6325 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6326 %{
6327 single_instruction;
6328 fixed_latency(3); // Maximum latency for 64 bit mul
6329 dst : WR(write);
6330 src1 : ISS(read);
6331 src2 : ISS(read);
6332 src3 : ISS(read);
6333 INS01 : ISS;
6334 MAC : WR;
6335 %}
6336
6337 //------- Divide pipeline operations --------------------
6338
6339 // Eg. SDIV w0, w1, w2
6340 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6341 %{
6342 single_instruction;
6343 fixed_latency(8); // Maximum latency for 32 bit divide
6344 dst : WR(write);
6345 src1 : ISS(read);
6346 src2 : ISS(read);
6347 INS0 : ISS; // Can only dual issue as instruction 0
6348 DIV : WR;
6349 %}
6350
6351 // Eg. SDIV x0, x1, x2
6352 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6353 %{
6354 single_instruction;
6355 fixed_latency(16); // Maximum latency for 64 bit divide
6356 dst : WR(write);
6357 src1 : ISS(read);
6358 src2 : ISS(read);
6359 INS0 : ISS; // Can only dual issue as instruction 0
6360 DIV : WR;
6361 %}
6362
6363 //------- Load pipeline operations ------------------------
6364
6365 // Load - prefetch
6366 // Eg. PFRM <mem>
6367 pipe_class iload_prefetch(memory mem)
6368 %{
6369 single_instruction;
6370 mem : ISS(read);
6371 INS01 : ISS;
6372 LDST : WR;
6373 %}
6374
6375 // Load - reg, mem
6376 // Eg. LDR x0, <mem>
6377 pipe_class iload_reg_mem(iRegI dst, memory mem)
6378 %{
6379 single_instruction;
6380 dst : WR(write);
6381 mem : ISS(read);
6382 INS01 : ISS;
6383 LDST : WR;
6384 %}
6385
6386 // Load - reg, reg
6387 // Eg. LDR x0, [sp, x1]
6388 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6389 %{
6390 single_instruction;
6391 dst : WR(write);
6392 src : ISS(read);
6393 INS01 : ISS;
6394 LDST : WR;
6395 %}
6396
6397 //------- Store pipeline operations -----------------------
6398
6399 // Store - zr, mem
6400 // Eg. STR zr, <mem>
6401 pipe_class istore_mem(memory mem)
6402 %{
6403 single_instruction;
6404 mem : ISS(read);
6405 INS01 : ISS;
6406 LDST : WR;
6407 %}
6408
6409 // Store - reg, mem
6410 // Eg. STR x0, <mem>
6411 pipe_class istore_reg_mem(iRegI src, memory mem)
6412 %{
6413 single_instruction;
6414 mem : ISS(read);
6415 src : EX2(read);
6416 INS01 : ISS;
6417 LDST : WR;
6418 %}
6419
6420 // Store - reg, reg
6421 // Eg. STR x0, [sp, x1]
6422 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6423 %{
6424 single_instruction;
6425 dst : ISS(read);
6426 src : EX2(read);
6427 INS01 : ISS;
6428 LDST : WR;
6429 %}
6430
6431 //------- Store pipeline operations -----------------------
6432
6433 // Branch
6434 pipe_class pipe_branch()
6435 %{
6436 single_instruction;
6437 INS01 : ISS;
6438 BRANCH : EX1;
6439 %}
6440
6441 // Conditional branch
6442 pipe_class pipe_branch_cond(rFlagsReg cr)
6443 %{
6444 single_instruction;
6445 cr : EX1(read);
6446 INS01 : ISS;
6447 BRANCH : EX1;
6448 %}
6449
6450 // Compare & Branch
6451 // EG. CBZ/CBNZ
6452 pipe_class pipe_cmp_branch(iRegI op1)
6453 %{
6454 single_instruction;
6455 op1 : EX1(read);
6456 INS01 : ISS;
6457 BRANCH : EX1;
6458 %}
6459
6460 //------- Synchronisation operations ----------------------
6461
6462 // Any operation requiring serialization.
6463 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6464 pipe_class pipe_serial()
6465 %{
6466 single_instruction;
6467 force_serialization;
6468 fixed_latency(16);
6469 INS01 : ISS(2); // Cannot dual issue with any other instruction
6470 LDST : WR;
6471 %}
6472
6473 // Generic big/slow expanded idiom - also serialized
6474 pipe_class pipe_slow()
6475 %{
6476 instruction_count(10);
6477 multiple_bundles;
6478 force_serialization;
6479 fixed_latency(16);
6480 INS01 : ISS(2); // Cannot dual issue with any other instruction
6481 LDST : WR;
6482 %}
6483
6484 // Empty pipeline class
6485 pipe_class pipe_class_empty()
6486 %{
6487 single_instruction;
6488 fixed_latency(0);
6489 %}
6490
6491 // Default pipeline class.
6492 pipe_class pipe_class_default()
6493 %{
6494 single_instruction;
6495 fixed_latency(2);
6496 %}
6497
6498 // Pipeline class for compares.
6499 pipe_class pipe_class_compare()
6500 %{
6501 single_instruction;
6502 fixed_latency(16);
6503 %}
6504
6505 // Pipeline class for memory operations.
6506 pipe_class pipe_class_memory()
6507 %{
6508 single_instruction;
6509 fixed_latency(16);
6510 %}
6511
6512 // Pipeline class for call.
6513 pipe_class pipe_class_call()
6514 %{
6515 single_instruction;
6516 fixed_latency(100);
6517 %}
6518
6519 // Define the class for the Nop node.
6520 define %{
6521 MachNop = pipe_class_empty;
6522 %}
6523
6524 %}
6525 //----------INSTRUCTIONS-------------------------------------------------------
6526 //
6527 // match -- States which machine-independent subtree may be replaced
6528 // by this instruction.
6529 // ins_cost -- The estimated cost of this instruction is used by instruction
6530 // selection to identify a minimum cost tree of machine
6531 // instructions that matches a tree of machine-independent
6532 // instructions.
6533 // format -- A string providing the disassembly for this instruction.
6534 // The value of an instruction's operand may be inserted
6535 // by referring to it with a '$' prefix.
6536 // opcode -- Three instruction opcodes may be provided. These are referred
6537 // to within an encode class as $primary, $secondary, and $tertiary
6538 // rrspectively. The primary opcode is commonly used to
6539 // indicate the type of machine instruction, while secondary
6540 // and tertiary are often used for prefix options or addressing
6541 // modes.
6542 // ins_encode -- A list of encode classes with parameters. The encode class
6543 // name must have been defined in an 'enc_class' specification
6544 // in the encode section of the architecture description.
6545
6546 // ============================================================================
6547 // Memory (Load/Store) Instructions
6548
6549 // Load Instructions
6550
6551 // Load Byte (8 bit signed)
6552 instruct loadB(iRegINoSp dst, memory1 mem)
6553 %{
6554 match(Set dst (LoadB mem));
6555 predicate(!needs_acquiring_load(n));
6556
6557 ins_cost(4 * INSN_COST);
6558 format %{ "ldrsbw $dst, $mem\t# byte" %}
6559
6560 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6561
6562 ins_pipe(iload_reg_mem);
6563 %}
6564
6565 // Load Byte (8 bit signed) into long
6566 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6567 %{
6568 match(Set dst (ConvI2L (LoadB mem)));
6569 predicate(!needs_acquiring_load(n->in(1)));
6570
6571 ins_cost(4 * INSN_COST);
6572 format %{ "ldrsb $dst, $mem\t# byte" %}
6573
6574 ins_encode(aarch64_enc_ldrsb(dst, mem));
6575
6576 ins_pipe(iload_reg_mem);
6577 %}
6578
6579 // Load Byte (8 bit unsigned)
6580 instruct loadUB(iRegINoSp dst, memory1 mem)
6581 %{
6582 match(Set dst (LoadUB mem));
6583 predicate(!needs_acquiring_load(n));
6584
6585 ins_cost(4 * INSN_COST);
6586 format %{ "ldrbw $dst, $mem\t# byte" %}
6587
6588 ins_encode(aarch64_enc_ldrb(dst, mem));
6589
6590 ins_pipe(iload_reg_mem);
6591 %}
6592
6593 // Load Byte (8 bit unsigned) into long
6594 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6595 %{
6596 match(Set dst (ConvI2L (LoadUB mem)));
6597 predicate(!needs_acquiring_load(n->in(1)));
6598
6599 ins_cost(4 * INSN_COST);
6600 format %{ "ldrb $dst, $mem\t# byte" %}
6601
6602 ins_encode(aarch64_enc_ldrb(dst, mem));
6603
6604 ins_pipe(iload_reg_mem);
6605 %}
6606
6607 // Load Short (16 bit signed)
6608 instruct loadS(iRegINoSp dst, memory2 mem)
6609 %{
6610 match(Set dst (LoadS mem));
6611 predicate(!needs_acquiring_load(n));
6612
6613 ins_cost(4 * INSN_COST);
6614 format %{ "ldrshw $dst, $mem\t# short" %}
6615
6616 ins_encode(aarch64_enc_ldrshw(dst, mem));
6617
6618 ins_pipe(iload_reg_mem);
6619 %}
6620
6621 // Load Short (16 bit signed) into long
6622 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6623 %{
6624 match(Set dst (ConvI2L (LoadS mem)));
6625 predicate(!needs_acquiring_load(n->in(1)));
6626
6627 ins_cost(4 * INSN_COST);
6628 format %{ "ldrsh $dst, $mem\t# short" %}
6629
6630 ins_encode(aarch64_enc_ldrsh(dst, mem));
6631
6632 ins_pipe(iload_reg_mem);
6633 %}
6634
6635 // Load Char (16 bit unsigned)
6636 instruct loadUS(iRegINoSp dst, memory2 mem)
6637 %{
6638 match(Set dst (LoadUS mem));
6639 predicate(!needs_acquiring_load(n));
6640
6641 ins_cost(4 * INSN_COST);
6642 format %{ "ldrh $dst, $mem\t# short" %}
6643
6644 ins_encode(aarch64_enc_ldrh(dst, mem));
6645
6646 ins_pipe(iload_reg_mem);
6647 %}
6648
6649 // Load Short/Char (16 bit unsigned) into long
6650 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6651 %{
6652 match(Set dst (ConvI2L (LoadUS mem)));
6653 predicate(!needs_acquiring_load(n->in(1)));
6654
6655 ins_cost(4 * INSN_COST);
6656 format %{ "ldrh $dst, $mem\t# short" %}
6657
6658 ins_encode(aarch64_enc_ldrh(dst, mem));
6659
6660 ins_pipe(iload_reg_mem);
6661 %}
6662
6663 // Load Integer (32 bit signed)
6664 instruct loadI(iRegINoSp dst, memory4 mem)
6665 %{
6666 match(Set dst (LoadI mem));
6667 predicate(!needs_acquiring_load(n));
6668
6669 ins_cost(4 * INSN_COST);
6670 format %{ "ldrw $dst, $mem\t# int" %}
6671
6672 ins_encode(aarch64_enc_ldrw(dst, mem));
6673
6674 ins_pipe(iload_reg_mem);
6675 %}
6676
6677 // Load Integer (32 bit signed) into long
6678 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6679 %{
6680 match(Set dst (ConvI2L (LoadI mem)));
6681 predicate(!needs_acquiring_load(n->in(1)));
6682
6683 ins_cost(4 * INSN_COST);
6684 format %{ "ldrsw $dst, $mem\t# int" %}
6685
6686 ins_encode(aarch64_enc_ldrsw(dst, mem));
6687
6688 ins_pipe(iload_reg_mem);
6689 %}
6690
6691 // Load Integer (32 bit unsigned) into long
6692 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6693 %{
6694 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6695 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6696
6697 ins_cost(4 * INSN_COST);
6698 format %{ "ldrw $dst, $mem\t# int" %}
6699
6700 ins_encode(aarch64_enc_ldrw(dst, mem));
6701
6702 ins_pipe(iload_reg_mem);
6703 %}
6704
6705 // Load Long (64 bit signed)
6706 instruct loadL(iRegLNoSp dst, memory8 mem)
6707 %{
6708 match(Set dst (LoadL mem));
6709 predicate(!needs_acquiring_load(n));
6710
6711 ins_cost(4 * INSN_COST);
6712 format %{ "ldr $dst, $mem\t# int" %}
6713
6714 ins_encode(aarch64_enc_ldr(dst, mem));
6715
6716 ins_pipe(iload_reg_mem);
6717 %}
6718
6719 // Load Range
6720 instruct loadRange(iRegINoSp dst, memory4 mem)
6721 %{
6722 match(Set dst (LoadRange mem));
6723
6724 ins_cost(4 * INSN_COST);
6725 format %{ "ldrw $dst, $mem\t# range" %}
6726
6727 ins_encode(aarch64_enc_ldrw(dst, mem));
6728
6729 ins_pipe(iload_reg_mem);
6730 %}
6731
6732 // Load Pointer
6733 instruct loadP(iRegPNoSp dst, memory8 mem)
6734 %{
6735 match(Set dst (LoadP mem));
6736 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6737
6738 ins_cost(4 * INSN_COST);
6739 format %{ "ldr $dst, $mem\t# ptr" %}
6740
6741 ins_encode(aarch64_enc_ldr(dst, mem));
6742
6743 ins_pipe(iload_reg_mem);
6744 %}
6745
6746 // Load Compressed Pointer
6747 instruct loadN(iRegNNoSp dst, memory4 mem)
6748 %{
6749 match(Set dst (LoadN mem));
6750 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6751
6752 ins_cost(4 * INSN_COST);
6753 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6754
6755 ins_encode(aarch64_enc_ldrw(dst, mem));
6756
6757 ins_pipe(iload_reg_mem);
6758 %}
6759
6760 // Load Klass Pointer
6761 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6762 %{
6763 match(Set dst (LoadKlass mem));
6764 predicate(!needs_acquiring_load(n));
6765
6766 ins_cost(4 * INSN_COST);
6767 format %{ "ldr $dst, $mem\t# class" %}
6768
6769 ins_encode(aarch64_enc_ldr(dst, mem));
6770
6771 ins_pipe(iload_reg_mem);
6772 %}
6773
6774 // Load Narrow Klass Pointer
6775 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6776 %{
6777 match(Set dst (LoadNKlass mem));
6778 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6779
6780 ins_cost(4 * INSN_COST);
6781 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6782
6783 ins_encode(aarch64_enc_ldrw(dst, mem));
6784
6785 ins_pipe(iload_reg_mem);
6786 %}
6787
6788 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory_noindex mem)
6789 %{
6790 match(Set dst (LoadNKlass mem));
6791 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6792
6793 ins_cost(4 * INSN_COST);
6794 format %{
6795 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6796 "lsrw $dst, $dst, markWord::klass_shift"
6797 %}
6798 ins_encode %{
6799 assert($mem$$index$$Register == noreg, "must not have indexed address");
6800 // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
6801 // obj-start, so that we can load from the object's mark-word instead.
6802 __ ldrw($dst$$Register, Address($mem$$base$$Register, $mem$$disp - Type::klass_offset()));
6803 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift);
6804 %}
6805 ins_pipe(iload_reg_mem);
6806 %}
6807
6808 // Load Float
6809 instruct loadF(vRegF dst, memory4 mem)
6810 %{
6811 match(Set dst (LoadF mem));
6812 predicate(!needs_acquiring_load(n));
6813
6814 ins_cost(4 * INSN_COST);
6815 format %{ "ldrs $dst, $mem\t# float" %}
6816
6817 ins_encode( aarch64_enc_ldrs(dst, mem) );
6818
6819 ins_pipe(pipe_class_memory);
6820 %}
6821
6822 // Load Double
6823 instruct loadD(vRegD dst, memory8 mem)
6824 %{
6825 match(Set dst (LoadD mem));
6826 predicate(!needs_acquiring_load(n));
6827
6828 ins_cost(4 * INSN_COST);
6829 format %{ "ldrd $dst, $mem\t# double" %}
6830
6831 ins_encode( aarch64_enc_ldrd(dst, mem) );
6832
6833 ins_pipe(pipe_class_memory);
6834 %}
6835
6836
6837 // Load Int Constant
6838 instruct loadConI(iRegINoSp dst, immI src)
6839 %{
6840 match(Set dst src);
6841
6842 ins_cost(INSN_COST);
6843 format %{ "mov $dst, $src\t# int" %}
6844
6845 ins_encode( aarch64_enc_movw_imm(dst, src) );
6846
6847 ins_pipe(ialu_imm);
6848 %}
6849
6850 // Load Long Constant
6851 instruct loadConL(iRegLNoSp dst, immL src)
6852 %{
6853 match(Set dst src);
6854
6855 ins_cost(INSN_COST);
6856 format %{ "mov $dst, $src\t# long" %}
6857
6858 ins_encode( aarch64_enc_mov_imm(dst, src) );
6859
6860 ins_pipe(ialu_imm);
6861 %}
6862
6863 // Load Pointer Constant
6864
6865 instruct loadConP(iRegPNoSp dst, immP con)
6866 %{
6867 match(Set dst con);
6868
6869 ins_cost(INSN_COST * 4);
6870 format %{
6871 "mov $dst, $con\t# ptr\n\t"
6872 %}
6873
6874 ins_encode(aarch64_enc_mov_p(dst, con));
6875
6876 ins_pipe(ialu_imm);
6877 %}
6878
6879 // Load Null Pointer Constant
6880
6881 instruct loadConP0(iRegPNoSp dst, immP0 con)
6882 %{
6883 match(Set dst con);
6884
6885 ins_cost(INSN_COST);
6886 format %{ "mov $dst, $con\t# nullptr ptr" %}
6887
6888 ins_encode(aarch64_enc_mov_p0(dst, con));
6889
6890 ins_pipe(ialu_imm);
6891 %}
6892
6893 // Load Pointer Constant One
6894
6895 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6896 %{
6897 match(Set dst con);
6898
6899 ins_cost(INSN_COST);
6900 format %{ "mov $dst, $con\t# nullptr ptr" %}
6901
6902 ins_encode(aarch64_enc_mov_p1(dst, con));
6903
6904 ins_pipe(ialu_imm);
6905 %}
6906
6907 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6908 %{
6909 match(Set dst con);
6910
6911 ins_cost(INSN_COST);
6912 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6913
6914 ins_encode %{
6915 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6916 %}
6917
6918 ins_pipe(ialu_imm);
6919 %}
6920
6921 // Load Narrow Pointer Constant
6922
6923 instruct loadConN(iRegNNoSp dst, immN con)
6924 %{
6925 match(Set dst con);
6926
6927 ins_cost(INSN_COST * 4);
6928 format %{ "mov $dst, $con\t# compressed ptr" %}
6929
6930 ins_encode(aarch64_enc_mov_n(dst, con));
6931
6932 ins_pipe(ialu_imm);
6933 %}
6934
6935 // Load Narrow Null Pointer Constant
6936
6937 instruct loadConN0(iRegNNoSp dst, immN0 con)
6938 %{
6939 match(Set dst con);
6940
6941 ins_cost(INSN_COST);
6942 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6943
6944 ins_encode(aarch64_enc_mov_n0(dst, con));
6945
6946 ins_pipe(ialu_imm);
6947 %}
6948
6949 // Load Narrow Klass Constant
6950
6951 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6952 %{
6953 match(Set dst con);
6954
6955 ins_cost(INSN_COST);
6956 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6957
6958 ins_encode(aarch64_enc_mov_nk(dst, con));
6959
6960 ins_pipe(ialu_imm);
6961 %}
6962
6963 // Load Packed Float Constant
6964
6965 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6966 match(Set dst con);
6967 ins_cost(INSN_COST * 4);
6968 format %{ "fmovs $dst, $con"%}
6969 ins_encode %{
6970 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6971 %}
6972
6973 ins_pipe(fp_imm_s);
6974 %}
6975
6976 // Load Float Constant
6977
6978 instruct loadConF(vRegF dst, immF con) %{
6979 match(Set dst con);
6980
6981 ins_cost(INSN_COST * 4);
6982
6983 format %{
6984 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6985 %}
6986
6987 ins_encode %{
6988 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6989 %}
6990
6991 ins_pipe(fp_load_constant_s);
6992 %}
6993
6994 // Load Packed Double Constant
6995
6996 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6997 match(Set dst con);
6998 ins_cost(INSN_COST);
6999 format %{ "fmovd $dst, $con"%}
7000 ins_encode %{
7001 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7002 %}
7003
7004 ins_pipe(fp_imm_d);
7005 %}
7006
7007 // Load Double Constant
7008
7009 instruct loadConD(vRegD dst, immD con) %{
7010 match(Set dst con);
7011
7012 ins_cost(INSN_COST * 5);
7013 format %{
7014 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7015 %}
7016
7017 ins_encode %{
7018 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7019 %}
7020
7021 ins_pipe(fp_load_constant_d);
7022 %}
7023
7024 // Load Half Float Constant
7025 instruct loadConH(vRegF dst, immH con) %{
7026 match(Set dst con);
7027 format %{ "mov rscratch1, $con\n\t"
7028 "fmov $dst, rscratch1"
7029 %}
7030 ins_encode %{
7031 __ movw(rscratch1, (uint32_t)$con$$constant);
7032 __ fmovs($dst$$FloatRegister, rscratch1);
7033 %}
7034 ins_pipe(pipe_class_default);
7035 %}
7036
7037 // Store Instructions
7038
7039 // Store Byte
7040 instruct storeB(iRegIorL2I src, memory1 mem)
7041 %{
7042 match(Set mem (StoreB mem src));
7043 predicate(!needs_releasing_store(n));
7044
7045 ins_cost(INSN_COST);
7046 format %{ "strb $src, $mem\t# byte" %}
7047
7048 ins_encode(aarch64_enc_strb(src, mem));
7049
7050 ins_pipe(istore_reg_mem);
7051 %}
7052
7053
7054 instruct storeimmB0(immI0 zero, memory1 mem)
7055 %{
7056 match(Set mem (StoreB mem zero));
7057 predicate(!needs_releasing_store(n));
7058
7059 ins_cost(INSN_COST);
7060 format %{ "strb rscractch2, $mem\t# byte" %}
7061
7062 ins_encode(aarch64_enc_strb0(mem));
7063
7064 ins_pipe(istore_mem);
7065 %}
7066
7067 // Store Char/Short
7068 instruct storeC(iRegIorL2I src, memory2 mem)
7069 %{
7070 match(Set mem (StoreC mem src));
7071 predicate(!needs_releasing_store(n));
7072
7073 ins_cost(INSN_COST);
7074 format %{ "strh $src, $mem\t# short" %}
7075
7076 ins_encode(aarch64_enc_strh(src, mem));
7077
7078 ins_pipe(istore_reg_mem);
7079 %}
7080
7081 instruct storeimmC0(immI0 zero, memory2 mem)
7082 %{
7083 match(Set mem (StoreC mem zero));
7084 predicate(!needs_releasing_store(n));
7085
7086 ins_cost(INSN_COST);
7087 format %{ "strh zr, $mem\t# short" %}
7088
7089 ins_encode(aarch64_enc_strh0(mem));
7090
7091 ins_pipe(istore_mem);
7092 %}
7093
7094 // Store Integer
7095
7096 instruct storeI(iRegIorL2I src, memory4 mem)
7097 %{
7098 match(Set mem(StoreI mem src));
7099 predicate(!needs_releasing_store(n));
7100
7101 ins_cost(INSN_COST);
7102 format %{ "strw $src, $mem\t# int" %}
7103
7104 ins_encode(aarch64_enc_strw(src, mem));
7105
7106 ins_pipe(istore_reg_mem);
7107 %}
7108
7109 instruct storeimmI0(immI0 zero, memory4 mem)
7110 %{
7111 match(Set mem(StoreI mem zero));
7112 predicate(!needs_releasing_store(n));
7113
7114 ins_cost(INSN_COST);
7115 format %{ "strw zr, $mem\t# int" %}
7116
7117 ins_encode(aarch64_enc_strw0(mem));
7118
7119 ins_pipe(istore_mem);
7120 %}
7121
7122 // Store Long (64 bit signed)
7123 instruct storeL(iRegL src, memory8 mem)
7124 %{
7125 match(Set mem (StoreL mem src));
7126 predicate(!needs_releasing_store(n));
7127
7128 ins_cost(INSN_COST);
7129 format %{ "str $src, $mem\t# int" %}
7130
7131 ins_encode(aarch64_enc_str(src, mem));
7132
7133 ins_pipe(istore_reg_mem);
7134 %}
7135
7136 // Store Long (64 bit signed)
7137 instruct storeimmL0(immL0 zero, memory8 mem)
7138 %{
7139 match(Set mem (StoreL mem zero));
7140 predicate(!needs_releasing_store(n));
7141
7142 ins_cost(INSN_COST);
7143 format %{ "str zr, $mem\t# int" %}
7144
7145 ins_encode(aarch64_enc_str0(mem));
7146
7147 ins_pipe(istore_mem);
7148 %}
7149
7150 // Store Pointer
7151 instruct storeP(iRegP src, memory8 mem)
7152 %{
7153 match(Set mem (StoreP mem src));
7154 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7155
7156 ins_cost(INSN_COST);
7157 format %{ "str $src, $mem\t# ptr" %}
7158
7159 ins_encode(aarch64_enc_str(src, mem));
7160
7161 ins_pipe(istore_reg_mem);
7162 %}
7163
7164 // Store Pointer
7165 instruct storeimmP0(immP0 zero, memory8 mem)
7166 %{
7167 match(Set mem (StoreP mem zero));
7168 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7169
7170 ins_cost(INSN_COST);
7171 format %{ "str zr, $mem\t# ptr" %}
7172
7173 ins_encode(aarch64_enc_str0(mem));
7174
7175 ins_pipe(istore_mem);
7176 %}
7177
7178 // Store Compressed Pointer
7179 instruct storeN(iRegN src, memory4 mem)
7180 %{
7181 match(Set mem (StoreN mem src));
7182 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7183
7184 ins_cost(INSN_COST);
7185 format %{ "strw $src, $mem\t# compressed ptr" %}
7186
7187 ins_encode(aarch64_enc_strw(src, mem));
7188
7189 ins_pipe(istore_reg_mem);
7190 %}
7191
7192 instruct storeImmN0(immN0 zero, memory4 mem)
7193 %{
7194 match(Set mem (StoreN mem zero));
7195 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7196
7197 ins_cost(INSN_COST);
7198 format %{ "strw zr, $mem\t# compressed ptr" %}
7199
7200 ins_encode(aarch64_enc_strw0(mem));
7201
7202 ins_pipe(istore_mem);
7203 %}
7204
7205 // Store Float
7206 instruct storeF(vRegF src, memory4 mem)
7207 %{
7208 match(Set mem (StoreF mem src));
7209 predicate(!needs_releasing_store(n));
7210
7211 ins_cost(INSN_COST);
7212 format %{ "strs $src, $mem\t# float" %}
7213
7214 ins_encode( aarch64_enc_strs(src, mem) );
7215
7216 ins_pipe(pipe_class_memory);
7217 %}
7218
7219 // TODO
7220 // implement storeImmF0 and storeFImmPacked
7221
7222 // Store Double
7223 instruct storeD(vRegD src, memory8 mem)
7224 %{
7225 match(Set mem (StoreD mem src));
7226 predicate(!needs_releasing_store(n));
7227
7228 ins_cost(INSN_COST);
7229 format %{ "strd $src, $mem\t# double" %}
7230
7231 ins_encode( aarch64_enc_strd(src, mem) );
7232
7233 ins_pipe(pipe_class_memory);
7234 %}
7235
7236 // Store Compressed Klass Pointer
7237 instruct storeNKlass(iRegN src, memory4 mem)
7238 %{
7239 predicate(!needs_releasing_store(n));
7240 match(Set mem (StoreNKlass mem src));
7241
7242 ins_cost(INSN_COST);
7243 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7244
7245 ins_encode(aarch64_enc_strw(src, mem));
7246
7247 ins_pipe(istore_reg_mem);
7248 %}
7249
7250 // TODO
7251 // implement storeImmD0 and storeDImmPacked
7252
7253 // prefetch instructions
7254 // Must be safe to execute with invalid address (cannot fault).
7255
7256 instruct prefetchalloc( memory8 mem ) %{
7257 match(PrefetchAllocation mem);
7258
7259 ins_cost(INSN_COST);
7260 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7261
7262 ins_encode( aarch64_enc_prefetchw(mem) );
7263
7264 ins_pipe(iload_prefetch);
7265 %}
7266
7267 // ---------------- volatile loads and stores ----------------
7268
7269 // Load Byte (8 bit signed)
7270 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7271 %{
7272 match(Set dst (LoadB mem));
7273
7274 ins_cost(VOLATILE_REF_COST);
7275 format %{ "ldarsb $dst, $mem\t# byte" %}
7276
7277 ins_encode(aarch64_enc_ldarsb(dst, mem));
7278
7279 ins_pipe(pipe_serial);
7280 %}
7281
7282 // Load Byte (8 bit signed) into long
7283 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7284 %{
7285 match(Set dst (ConvI2L (LoadB mem)));
7286
7287 ins_cost(VOLATILE_REF_COST);
7288 format %{ "ldarsb $dst, $mem\t# byte" %}
7289
7290 ins_encode(aarch64_enc_ldarsb(dst, mem));
7291
7292 ins_pipe(pipe_serial);
7293 %}
7294
7295 // Load Byte (8 bit unsigned)
7296 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7297 %{
7298 match(Set dst (LoadUB mem));
7299
7300 ins_cost(VOLATILE_REF_COST);
7301 format %{ "ldarb $dst, $mem\t# byte" %}
7302
7303 ins_encode(aarch64_enc_ldarb(dst, mem));
7304
7305 ins_pipe(pipe_serial);
7306 %}
7307
7308 // Load Byte (8 bit unsigned) into long
7309 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7310 %{
7311 match(Set dst (ConvI2L (LoadUB mem)));
7312
7313 ins_cost(VOLATILE_REF_COST);
7314 format %{ "ldarb $dst, $mem\t# byte" %}
7315
7316 ins_encode(aarch64_enc_ldarb(dst, mem));
7317
7318 ins_pipe(pipe_serial);
7319 %}
7320
7321 // Load Short (16 bit signed)
7322 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7323 %{
7324 match(Set dst (LoadS mem));
7325
7326 ins_cost(VOLATILE_REF_COST);
7327 format %{ "ldarshw $dst, $mem\t# short" %}
7328
7329 ins_encode(aarch64_enc_ldarshw(dst, mem));
7330
7331 ins_pipe(pipe_serial);
7332 %}
7333
7334 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7335 %{
7336 match(Set dst (LoadUS mem));
7337
7338 ins_cost(VOLATILE_REF_COST);
7339 format %{ "ldarhw $dst, $mem\t# short" %}
7340
7341 ins_encode(aarch64_enc_ldarhw(dst, mem));
7342
7343 ins_pipe(pipe_serial);
7344 %}
7345
7346 // Load Short/Char (16 bit unsigned) into long
7347 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7348 %{
7349 match(Set dst (ConvI2L (LoadUS mem)));
7350
7351 ins_cost(VOLATILE_REF_COST);
7352 format %{ "ldarh $dst, $mem\t# short" %}
7353
7354 ins_encode(aarch64_enc_ldarh(dst, mem));
7355
7356 ins_pipe(pipe_serial);
7357 %}
7358
7359 // Load Short/Char (16 bit signed) into long
7360 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7361 %{
7362 match(Set dst (ConvI2L (LoadS mem)));
7363
7364 ins_cost(VOLATILE_REF_COST);
7365 format %{ "ldarh $dst, $mem\t# short" %}
7366
7367 ins_encode(aarch64_enc_ldarsh(dst, mem));
7368
7369 ins_pipe(pipe_serial);
7370 %}
7371
7372 // Load Integer (32 bit signed)
7373 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7374 %{
7375 match(Set dst (LoadI mem));
7376
7377 ins_cost(VOLATILE_REF_COST);
7378 format %{ "ldarw $dst, $mem\t# int" %}
7379
7380 ins_encode(aarch64_enc_ldarw(dst, mem));
7381
7382 ins_pipe(pipe_serial);
7383 %}
7384
7385 // Load Integer (32 bit unsigned) into long
7386 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7387 %{
7388 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7389
7390 ins_cost(VOLATILE_REF_COST);
7391 format %{ "ldarw $dst, $mem\t# int" %}
7392
7393 ins_encode(aarch64_enc_ldarw(dst, mem));
7394
7395 ins_pipe(pipe_serial);
7396 %}
7397
7398 // Load Long (64 bit signed)
7399 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7400 %{
7401 match(Set dst (LoadL mem));
7402
7403 ins_cost(VOLATILE_REF_COST);
7404 format %{ "ldar $dst, $mem\t# int" %}
7405
7406 ins_encode(aarch64_enc_ldar(dst, mem));
7407
7408 ins_pipe(pipe_serial);
7409 %}
7410
7411 // Load Pointer
7412 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7413 %{
7414 match(Set dst (LoadP mem));
7415 predicate(n->as_Load()->barrier_data() == 0);
7416
7417 ins_cost(VOLATILE_REF_COST);
7418 format %{ "ldar $dst, $mem\t# ptr" %}
7419
7420 ins_encode(aarch64_enc_ldar(dst, mem));
7421
7422 ins_pipe(pipe_serial);
7423 %}
7424
7425 // Load Compressed Pointer
7426 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7427 %{
7428 match(Set dst (LoadN mem));
7429 predicate(n->as_Load()->barrier_data() == 0);
7430
7431 ins_cost(VOLATILE_REF_COST);
7432 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7433
7434 ins_encode(aarch64_enc_ldarw(dst, mem));
7435
7436 ins_pipe(pipe_serial);
7437 %}
7438
7439 // Load Float
7440 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7441 %{
7442 match(Set dst (LoadF mem));
7443
7444 ins_cost(VOLATILE_REF_COST);
7445 format %{ "ldars $dst, $mem\t# float" %}
7446
7447 ins_encode( aarch64_enc_fldars(dst, mem) );
7448
7449 ins_pipe(pipe_serial);
7450 %}
7451
7452 // Load Double
7453 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7454 %{
7455 match(Set dst (LoadD mem));
7456
7457 ins_cost(VOLATILE_REF_COST);
7458 format %{ "ldard $dst, $mem\t# double" %}
7459
7460 ins_encode( aarch64_enc_fldard(dst, mem) );
7461
7462 ins_pipe(pipe_serial);
7463 %}
7464
7465 // Store Byte
7466 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7467 %{
7468 match(Set mem (StoreB mem src));
7469
7470 ins_cost(VOLATILE_REF_COST);
7471 format %{ "stlrb $src, $mem\t# byte" %}
7472
7473 ins_encode(aarch64_enc_stlrb(src, mem));
7474
7475 ins_pipe(pipe_class_memory);
7476 %}
7477
7478 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7479 %{
7480 match(Set mem (StoreB mem zero));
7481
7482 ins_cost(VOLATILE_REF_COST);
7483 format %{ "stlrb zr, $mem\t# byte" %}
7484
7485 ins_encode(aarch64_enc_stlrb0(mem));
7486
7487 ins_pipe(pipe_class_memory);
7488 %}
7489
7490 // Store Char/Short
7491 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7492 %{
7493 match(Set mem (StoreC mem src));
7494
7495 ins_cost(VOLATILE_REF_COST);
7496 format %{ "stlrh $src, $mem\t# short" %}
7497
7498 ins_encode(aarch64_enc_stlrh(src, mem));
7499
7500 ins_pipe(pipe_class_memory);
7501 %}
7502
7503 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7504 %{
7505 match(Set mem (StoreC mem zero));
7506
7507 ins_cost(VOLATILE_REF_COST);
7508 format %{ "stlrh zr, $mem\t# short" %}
7509
7510 ins_encode(aarch64_enc_stlrh0(mem));
7511
7512 ins_pipe(pipe_class_memory);
7513 %}
7514
7515 // Store Integer
7516
7517 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7518 %{
7519 match(Set mem(StoreI mem src));
7520
7521 ins_cost(VOLATILE_REF_COST);
7522 format %{ "stlrw $src, $mem\t# int" %}
7523
7524 ins_encode(aarch64_enc_stlrw(src, mem));
7525
7526 ins_pipe(pipe_class_memory);
7527 %}
7528
7529 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7530 %{
7531 match(Set mem(StoreI mem zero));
7532
7533 ins_cost(VOLATILE_REF_COST);
7534 format %{ "stlrw zr, $mem\t# int" %}
7535
7536 ins_encode(aarch64_enc_stlrw0(mem));
7537
7538 ins_pipe(pipe_class_memory);
7539 %}
7540
7541 // Store Long (64 bit signed)
7542 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7543 %{
7544 match(Set mem (StoreL mem src));
7545
7546 ins_cost(VOLATILE_REF_COST);
7547 format %{ "stlr $src, $mem\t# int" %}
7548
7549 ins_encode(aarch64_enc_stlr(src, mem));
7550
7551 ins_pipe(pipe_class_memory);
7552 %}
7553
7554 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7555 %{
7556 match(Set mem (StoreL mem zero));
7557
7558 ins_cost(VOLATILE_REF_COST);
7559 format %{ "stlr zr, $mem\t# int" %}
7560
7561 ins_encode(aarch64_enc_stlr0(mem));
7562
7563 ins_pipe(pipe_class_memory);
7564 %}
7565
7566 // Store Pointer
7567 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7568 %{
7569 match(Set mem (StoreP mem src));
7570 predicate(n->as_Store()->barrier_data() == 0);
7571
7572 ins_cost(VOLATILE_REF_COST);
7573 format %{ "stlr $src, $mem\t# ptr" %}
7574
7575 ins_encode(aarch64_enc_stlr(src, mem));
7576
7577 ins_pipe(pipe_class_memory);
7578 %}
7579
7580 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7581 %{
7582 match(Set mem (StoreP mem zero));
7583 predicate(n->as_Store()->barrier_data() == 0);
7584
7585 ins_cost(VOLATILE_REF_COST);
7586 format %{ "stlr zr, $mem\t# ptr" %}
7587
7588 ins_encode(aarch64_enc_stlr0(mem));
7589
7590 ins_pipe(pipe_class_memory);
7591 %}
7592
7593 // Store Compressed Pointer
7594 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7595 %{
7596 match(Set mem (StoreN mem src));
7597 predicate(n->as_Store()->barrier_data() == 0);
7598
7599 ins_cost(VOLATILE_REF_COST);
7600 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7601
7602 ins_encode(aarch64_enc_stlrw(src, mem));
7603
7604 ins_pipe(pipe_class_memory);
7605 %}
7606
7607 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7608 %{
7609 match(Set mem (StoreN mem zero));
7610 predicate(n->as_Store()->barrier_data() == 0);
7611
7612 ins_cost(VOLATILE_REF_COST);
7613 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7614
7615 ins_encode(aarch64_enc_stlrw0(mem));
7616
7617 ins_pipe(pipe_class_memory);
7618 %}
7619
7620 // Store Float
7621 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7622 %{
7623 match(Set mem (StoreF mem src));
7624
7625 ins_cost(VOLATILE_REF_COST);
7626 format %{ "stlrs $src, $mem\t# float" %}
7627
7628 ins_encode( aarch64_enc_fstlrs(src, mem) );
7629
7630 ins_pipe(pipe_class_memory);
7631 %}
7632
7633 // TODO
7634 // implement storeImmF0 and storeFImmPacked
7635
7636 // Store Double
7637 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7638 %{
7639 match(Set mem (StoreD mem src));
7640
7641 ins_cost(VOLATILE_REF_COST);
7642 format %{ "stlrd $src, $mem\t# double" %}
7643
7644 ins_encode( aarch64_enc_fstlrd(src, mem) );
7645
7646 ins_pipe(pipe_class_memory);
7647 %}
7648
7649 // ---------------- end of volatile loads and stores ----------------
7650
7651 instruct cacheWB(indirect addr)
7652 %{
7653 predicate(VM_Version::supports_data_cache_line_flush());
7654 match(CacheWB addr);
7655
7656 ins_cost(100);
7657 format %{"cache wb $addr" %}
7658 ins_encode %{
7659 assert($addr->index_position() < 0, "should be");
7660 assert($addr$$disp == 0, "should be");
7661 __ cache_wb(Address($addr$$base$$Register, 0));
7662 %}
7663 ins_pipe(pipe_slow); // XXX
7664 %}
7665
7666 instruct cacheWBPreSync()
7667 %{
7668 predicate(VM_Version::supports_data_cache_line_flush());
7669 match(CacheWBPreSync);
7670
7671 ins_cost(100);
7672 format %{"cache wb presync" %}
7673 ins_encode %{
7674 __ cache_wbsync(true);
7675 %}
7676 ins_pipe(pipe_slow); // XXX
7677 %}
7678
7679 instruct cacheWBPostSync()
7680 %{
7681 predicate(VM_Version::supports_data_cache_line_flush());
7682 match(CacheWBPostSync);
7683
7684 ins_cost(100);
7685 format %{"cache wb postsync" %}
7686 ins_encode %{
7687 __ cache_wbsync(false);
7688 %}
7689 ins_pipe(pipe_slow); // XXX
7690 %}
7691
7692 // ============================================================================
7693 // BSWAP Instructions
7694
7695 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7696 match(Set dst (ReverseBytesI src));
7697
7698 ins_cost(INSN_COST);
7699 format %{ "revw $dst, $src" %}
7700
7701 ins_encode %{
7702 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7703 %}
7704
7705 ins_pipe(ialu_reg);
7706 %}
7707
7708 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7709 match(Set dst (ReverseBytesL src));
7710
7711 ins_cost(INSN_COST);
7712 format %{ "rev $dst, $src" %}
7713
7714 ins_encode %{
7715 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7716 %}
7717
7718 ins_pipe(ialu_reg);
7719 %}
7720
7721 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7722 match(Set dst (ReverseBytesUS src));
7723
7724 ins_cost(INSN_COST);
7725 format %{ "rev16w $dst, $src" %}
7726
7727 ins_encode %{
7728 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7729 %}
7730
7731 ins_pipe(ialu_reg);
7732 %}
7733
7734 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7735 match(Set dst (ReverseBytesS src));
7736
7737 ins_cost(INSN_COST);
7738 format %{ "rev16w $dst, $src\n\t"
7739 "sbfmw $dst, $dst, #0, #15" %}
7740
7741 ins_encode %{
7742 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7743 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7744 %}
7745
7746 ins_pipe(ialu_reg);
7747 %}
7748
7749 // ============================================================================
7750 // Zero Count Instructions
7751
7752 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7753 match(Set dst (CountLeadingZerosI src));
7754
7755 ins_cost(INSN_COST);
7756 format %{ "clzw $dst, $src" %}
7757 ins_encode %{
7758 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7759 %}
7760
7761 ins_pipe(ialu_reg);
7762 %}
7763
7764 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7765 match(Set dst (CountLeadingZerosL src));
7766
7767 ins_cost(INSN_COST);
7768 format %{ "clz $dst, $src" %}
7769 ins_encode %{
7770 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7771 %}
7772
7773 ins_pipe(ialu_reg);
7774 %}
7775
7776 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7777 match(Set dst (CountTrailingZerosI src));
7778
7779 ins_cost(INSN_COST * 2);
7780 format %{ "rbitw $dst, $src\n\t"
7781 "clzw $dst, $dst" %}
7782 ins_encode %{
7783 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7784 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7785 %}
7786
7787 ins_pipe(ialu_reg);
7788 %}
7789
7790 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7791 match(Set dst (CountTrailingZerosL src));
7792
7793 ins_cost(INSN_COST * 2);
7794 format %{ "rbit $dst, $src\n\t"
7795 "clz $dst, $dst" %}
7796 ins_encode %{
7797 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7798 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7799 %}
7800
7801 ins_pipe(ialu_reg);
7802 %}
7803
7804 //---------- Population Count Instructions -------------------------------------
7805 //
7806
7807 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7808 match(Set dst (PopCountI src));
7809 effect(TEMP tmp);
7810 ins_cost(INSN_COST * 13);
7811
7812 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7813 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7814 "addv $tmp, $tmp\t# vector (8B)\n\t"
7815 "mov $dst, $tmp\t# vector (1D)" %}
7816 ins_encode %{
7817 __ fmovs($tmp$$FloatRegister, $src$$Register);
7818 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7819 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7820 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7821 %}
7822
7823 ins_pipe(pipe_class_default);
7824 %}
7825
7826 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7827 match(Set dst (PopCountI (LoadI mem)));
7828 effect(TEMP tmp);
7829 ins_cost(INSN_COST * 13);
7830
7831 format %{ "ldrs $tmp, $mem\n\t"
7832 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7833 "addv $tmp, $tmp\t# vector (8B)\n\t"
7834 "mov $dst, $tmp\t# vector (1D)" %}
7835 ins_encode %{
7836 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7837 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7838 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7839 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7840 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7841 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7842 %}
7843
7844 ins_pipe(pipe_class_default);
7845 %}
7846
7847 // Note: Long.bitCount(long) returns an int.
7848 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7849 match(Set dst (PopCountL src));
7850 effect(TEMP tmp);
7851 ins_cost(INSN_COST * 13);
7852
7853 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7854 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7855 "addv $tmp, $tmp\t# vector (8B)\n\t"
7856 "mov $dst, $tmp\t# vector (1D)" %}
7857 ins_encode %{
7858 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7859 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7860 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7861 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7862 %}
7863
7864 ins_pipe(pipe_class_default);
7865 %}
7866
7867 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7868 match(Set dst (PopCountL (LoadL mem)));
7869 effect(TEMP tmp);
7870 ins_cost(INSN_COST * 13);
7871
7872 format %{ "ldrd $tmp, $mem\n\t"
7873 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7874 "addv $tmp, $tmp\t# vector (8B)\n\t"
7875 "mov $dst, $tmp\t# vector (1D)" %}
7876 ins_encode %{
7877 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7878 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7879 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7880 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7881 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7882 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7883 %}
7884
7885 ins_pipe(pipe_class_default);
7886 %}
7887
7888 // ============================================================================
7889 // VerifyVectorAlignment Instruction
7890
7891 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7892 match(Set addr (VerifyVectorAlignment addr mask));
7893 effect(KILL cr);
7894 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7895 ins_encode %{
7896 Label Lskip;
7897 // check if masked bits of addr are zero
7898 __ tst($addr$$Register, $mask$$constant);
7899 __ br(Assembler::EQ, Lskip);
7900 __ stop("verify_vector_alignment found a misaligned vector memory access");
7901 __ bind(Lskip);
7902 %}
7903 ins_pipe(pipe_slow);
7904 %}
7905
7906 // ============================================================================
7907 // MemBar Instruction
7908
7909 instruct load_fence() %{
7910 match(LoadFence);
7911 ins_cost(VOLATILE_REF_COST);
7912
7913 format %{ "load_fence" %}
7914
7915 ins_encode %{
7916 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7917 %}
7918 ins_pipe(pipe_serial);
7919 %}
7920
7921 instruct unnecessary_membar_acquire() %{
7922 predicate(unnecessary_acquire(n));
7923 match(MemBarAcquire);
7924 ins_cost(0);
7925
7926 format %{ "membar_acquire (elided)" %}
7927
7928 ins_encode %{
7929 __ block_comment("membar_acquire (elided)");
7930 %}
7931
7932 ins_pipe(pipe_class_empty);
7933 %}
7934
7935 instruct membar_acquire() %{
7936 match(MemBarAcquire);
7937 ins_cost(VOLATILE_REF_COST);
7938
7939 format %{ "membar_acquire\n\t"
7940 "dmb ishld" %}
7941
7942 ins_encode %{
7943 __ block_comment("membar_acquire");
7944 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7945 %}
7946
7947 ins_pipe(pipe_serial);
7948 %}
7949
7950
7951 instruct membar_acquire_lock() %{
7952 match(MemBarAcquireLock);
7953 ins_cost(VOLATILE_REF_COST);
7954
7955 format %{ "membar_acquire_lock (elided)" %}
7956
7957 ins_encode %{
7958 __ block_comment("membar_acquire_lock (elided)");
7959 %}
7960
7961 ins_pipe(pipe_serial);
7962 %}
7963
7964 instruct store_fence() %{
7965 match(StoreFence);
7966 ins_cost(VOLATILE_REF_COST);
7967
7968 format %{ "store_fence" %}
7969
7970 ins_encode %{
7971 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7972 %}
7973 ins_pipe(pipe_serial);
7974 %}
7975
7976 instruct unnecessary_membar_release() %{
7977 predicate(unnecessary_release(n));
7978 match(MemBarRelease);
7979 ins_cost(0);
7980
7981 format %{ "membar_release (elided)" %}
7982
7983 ins_encode %{
7984 __ block_comment("membar_release (elided)");
7985 %}
7986 ins_pipe(pipe_serial);
7987 %}
7988
7989 instruct membar_release() %{
7990 match(MemBarRelease);
7991 ins_cost(VOLATILE_REF_COST);
7992
7993 format %{ "membar_release\n\t"
7994 "dmb ishst\n\tdmb ishld" %}
7995
7996 ins_encode %{
7997 __ block_comment("membar_release");
7998 // These will be merged if AlwaysMergeDMB is enabled.
7999 __ membar(Assembler::StoreStore);
8000 __ membar(Assembler::LoadStore);
8001 %}
8002 ins_pipe(pipe_serial);
8003 %}
8004
8005 instruct membar_storestore() %{
8006 match(MemBarStoreStore);
8007 match(StoreStoreFence);
8008 ins_cost(VOLATILE_REF_COST);
8009
8010 format %{ "MEMBAR-store-store" %}
8011
8012 ins_encode %{
8013 __ membar(Assembler::StoreStore);
8014 %}
8015 ins_pipe(pipe_serial);
8016 %}
8017
8018 instruct membar_release_lock() %{
8019 match(MemBarReleaseLock);
8020 ins_cost(VOLATILE_REF_COST);
8021
8022 format %{ "membar_release_lock (elided)" %}
8023
8024 ins_encode %{
8025 __ block_comment("membar_release_lock (elided)");
8026 %}
8027
8028 ins_pipe(pipe_serial);
8029 %}
8030
8031 instruct membar_storeload() %{
8032 match(MemBarStoreLoad);
8033 ins_cost(VOLATILE_REF_COST*100);
8034
8035 format %{ "MEMBAR-store-load\n\t"
8036 "dmb ish" %}
8037
8038 ins_encode %{
8039 __ block_comment("membar_storeload");
8040 __ membar(Assembler::StoreLoad);
8041 %}
8042
8043 ins_pipe(pipe_serial);
8044 %}
8045
8046 instruct unnecessary_membar_volatile() %{
8047 predicate(unnecessary_volatile(n));
8048 match(MemBarVolatile);
8049 ins_cost(0);
8050
8051 format %{ "membar_volatile (elided)" %}
8052
8053 ins_encode %{
8054 __ block_comment("membar_volatile (elided)");
8055 %}
8056
8057 ins_pipe(pipe_serial);
8058 %}
8059
8060 instruct membar_volatile() %{
8061 match(MemBarVolatile);
8062 ins_cost(VOLATILE_REF_COST*100);
8063
8064 format %{ "membar_volatile\n\t"
8065 "dmb ish"%}
8066
8067 ins_encode %{
8068 __ block_comment("membar_volatile");
8069 __ membar(Assembler::StoreLoad);
8070 %}
8071
8072 ins_pipe(pipe_serial);
8073 %}
8074
8075 instruct membar_full() %{
8076 match(MemBarFull);
8077 ins_cost(VOLATILE_REF_COST*100);
8078
8079 format %{ "membar_full\n\t"
8080 "dmb ish" %}
8081 ins_encode %{
8082 __ block_comment("membar_full");
8083 __ membar(Assembler::AnyAny);
8084 %}
8085
8086 ins_pipe(pipe_serial);
8087 %}
8088
8089 // ============================================================================
8090 // Cast/Convert Instructions
8091
8092 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8093 match(Set dst (CastX2P src));
8094
8095 ins_cost(INSN_COST);
8096 format %{ "mov $dst, $src\t# long -> ptr" %}
8097
8098 ins_encode %{
8099 if ($dst$$reg != $src$$reg) {
8100 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8101 }
8102 %}
8103
8104 ins_pipe(ialu_reg);
8105 %}
8106
8107 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8108 match(Set dst (CastP2X src));
8109
8110 ins_cost(INSN_COST);
8111 format %{ "mov $dst, $src\t# ptr -> long" %}
8112
8113 ins_encode %{
8114 if ($dst$$reg != $src$$reg) {
8115 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8116 }
8117 %}
8118
8119 ins_pipe(ialu_reg);
8120 %}
8121
8122 // Convert oop into int for vectors alignment masking
8123 instruct convP2I(iRegINoSp dst, iRegP src) %{
8124 match(Set dst (ConvL2I (CastP2X src)));
8125
8126 ins_cost(INSN_COST);
8127 format %{ "movw $dst, $src\t# ptr -> int" %}
8128 ins_encode %{
8129 __ movw($dst$$Register, $src$$Register);
8130 %}
8131
8132 ins_pipe(ialu_reg);
8133 %}
8134
8135 // Convert compressed oop into int for vectors alignment masking
8136 // in case of 32bit oops (heap < 4Gb).
8137 instruct convN2I(iRegINoSp dst, iRegN src)
8138 %{
8139 predicate(CompressedOops::shift() == 0);
8140 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8141
8142 ins_cost(INSN_COST);
8143 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8144 ins_encode %{
8145 __ movw($dst$$Register, $src$$Register);
8146 %}
8147
8148 ins_pipe(ialu_reg);
8149 %}
8150
8151
8152 // Convert oop pointer into compressed form
8153 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8154 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8155 match(Set dst (EncodeP src));
8156 effect(KILL cr);
8157 ins_cost(INSN_COST * 3);
8158 format %{ "encode_heap_oop $dst, $src" %}
8159 ins_encode %{
8160 Register s = $src$$Register;
8161 Register d = $dst$$Register;
8162 __ encode_heap_oop(d, s);
8163 %}
8164 ins_pipe(ialu_reg);
8165 %}
8166
8167 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8168 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8169 match(Set dst (EncodeP src));
8170 ins_cost(INSN_COST * 3);
8171 format %{ "encode_heap_oop_not_null $dst, $src" %}
8172 ins_encode %{
8173 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8174 %}
8175 ins_pipe(ialu_reg);
8176 %}
8177
8178 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8179 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8180 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8181 match(Set dst (DecodeN src));
8182 ins_cost(INSN_COST * 3);
8183 format %{ "decode_heap_oop $dst, $src" %}
8184 ins_encode %{
8185 Register s = $src$$Register;
8186 Register d = $dst$$Register;
8187 __ decode_heap_oop(d, s);
8188 %}
8189 ins_pipe(ialu_reg);
8190 %}
8191
8192 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8193 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8194 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8195 match(Set dst (DecodeN src));
8196 ins_cost(INSN_COST * 3);
8197 format %{ "decode_heap_oop_not_null $dst, $src" %}
8198 ins_encode %{
8199 Register s = $src$$Register;
8200 Register d = $dst$$Register;
8201 __ decode_heap_oop_not_null(d, s);
8202 %}
8203 ins_pipe(ialu_reg);
8204 %}
8205
8206 // n.b. AArch64 implementations of encode_klass_not_null and
8207 // decode_klass_not_null do not modify the flags register so, unlike
8208 // Intel, we don't kill CR as a side effect here
8209
8210 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8211 match(Set dst (EncodePKlass src));
8212
8213 ins_cost(INSN_COST * 3);
8214 format %{ "encode_klass_not_null $dst,$src" %}
8215
8216 ins_encode %{
8217 Register src_reg = as_Register($src$$reg);
8218 Register dst_reg = as_Register($dst$$reg);
8219 __ encode_klass_not_null(dst_reg, src_reg);
8220 %}
8221
8222 ins_pipe(ialu_reg);
8223 %}
8224
8225 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8226 match(Set dst (DecodeNKlass src));
8227
8228 ins_cost(INSN_COST * 3);
8229 format %{ "decode_klass_not_null $dst,$src" %}
8230
8231 ins_encode %{
8232 Register src_reg = as_Register($src$$reg);
8233 Register dst_reg = as_Register($dst$$reg);
8234 if (dst_reg != src_reg) {
8235 __ decode_klass_not_null(dst_reg, src_reg);
8236 } else {
8237 __ decode_klass_not_null(dst_reg);
8238 }
8239 %}
8240
8241 ins_pipe(ialu_reg);
8242 %}
8243
8244 instruct checkCastPP(iRegPNoSp dst)
8245 %{
8246 match(Set dst (CheckCastPP dst));
8247
8248 size(0);
8249 format %{ "# checkcastPP of $dst" %}
8250 ins_encode(/* empty encoding */);
8251 ins_pipe(pipe_class_empty);
8252 %}
8253
8254 instruct castPP(iRegPNoSp dst)
8255 %{
8256 match(Set dst (CastPP dst));
8257
8258 size(0);
8259 format %{ "# castPP of $dst" %}
8260 ins_encode(/* empty encoding */);
8261 ins_pipe(pipe_class_empty);
8262 %}
8263
8264 instruct castII(iRegI dst)
8265 %{
8266 predicate(VerifyConstraintCasts == 0);
8267 match(Set dst (CastII dst));
8268
8269 size(0);
8270 format %{ "# castII of $dst" %}
8271 ins_encode(/* empty encoding */);
8272 ins_cost(0);
8273 ins_pipe(pipe_class_empty);
8274 %}
8275
8276 instruct castII_checked(iRegI dst, rFlagsReg cr)
8277 %{
8278 predicate(VerifyConstraintCasts > 0);
8279 match(Set dst (CastII dst));
8280 effect(KILL cr);
8281
8282 format %{ "# castII_checked of $dst" %}
8283 ins_encode %{
8284 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8285 %}
8286 ins_pipe(pipe_slow);
8287 %}
8288
8289 instruct castLL(iRegL dst)
8290 %{
8291 predicate(VerifyConstraintCasts == 0);
8292 match(Set dst (CastLL dst));
8293
8294 size(0);
8295 format %{ "# castLL of $dst" %}
8296 ins_encode(/* empty encoding */);
8297 ins_cost(0);
8298 ins_pipe(pipe_class_empty);
8299 %}
8300
8301 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8302 %{
8303 predicate(VerifyConstraintCasts > 0);
8304 match(Set dst (CastLL dst));
8305 effect(KILL cr);
8306
8307 format %{ "# castLL_checked of $dst" %}
8308 ins_encode %{
8309 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8310 %}
8311 ins_pipe(pipe_slow);
8312 %}
8313
8314 instruct castHH(vRegF dst)
8315 %{
8316 match(Set dst (CastHH dst));
8317 size(0);
8318 format %{ "# castHH of $dst" %}
8319 ins_encode(/* empty encoding */);
8320 ins_cost(0);
8321 ins_pipe(pipe_class_empty);
8322 %}
8323
8324 instruct castFF(vRegF dst)
8325 %{
8326 match(Set dst (CastFF dst));
8327
8328 size(0);
8329 format %{ "# castFF of $dst" %}
8330 ins_encode(/* empty encoding */);
8331 ins_cost(0);
8332 ins_pipe(pipe_class_empty);
8333 %}
8334
8335 instruct castDD(vRegD dst)
8336 %{
8337 match(Set dst (CastDD dst));
8338
8339 size(0);
8340 format %{ "# castDD of $dst" %}
8341 ins_encode(/* empty encoding */);
8342 ins_cost(0);
8343 ins_pipe(pipe_class_empty);
8344 %}
8345
8346 instruct castVV(vReg dst)
8347 %{
8348 match(Set dst (CastVV dst));
8349
8350 size(0);
8351 format %{ "# castVV of $dst" %}
8352 ins_encode(/* empty encoding */);
8353 ins_cost(0);
8354 ins_pipe(pipe_class_empty);
8355 %}
8356
8357 instruct castVVMask(pRegGov dst)
8358 %{
8359 match(Set dst (CastVV dst));
8360
8361 size(0);
8362 format %{ "# castVV of $dst" %}
8363 ins_encode(/* empty encoding */);
8364 ins_cost(0);
8365 ins_pipe(pipe_class_empty);
8366 %}
8367
8368 // Manifest a CmpU result in an integer register.
8369 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8370 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8371 %{
8372 match(Set dst (CmpU3 src1 src2));
8373 effect(KILL flags);
8374
8375 ins_cost(INSN_COST * 3);
8376 format %{
8377 "cmpw $src1, $src2\n\t"
8378 "csetw $dst, ne\n\t"
8379 "cnegw $dst, lo\t# CmpU3(reg)"
8380 %}
8381 ins_encode %{
8382 __ cmpw($src1$$Register, $src2$$Register);
8383 __ csetw($dst$$Register, Assembler::NE);
8384 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8385 %}
8386
8387 ins_pipe(pipe_class_default);
8388 %}
8389
8390 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8391 %{
8392 match(Set dst (CmpU3 src1 src2));
8393 effect(KILL flags);
8394
8395 ins_cost(INSN_COST * 3);
8396 format %{
8397 "subsw zr, $src1, $src2\n\t"
8398 "csetw $dst, ne\n\t"
8399 "cnegw $dst, lo\t# CmpU3(imm)"
8400 %}
8401 ins_encode %{
8402 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8403 __ csetw($dst$$Register, Assembler::NE);
8404 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8405 %}
8406
8407 ins_pipe(pipe_class_default);
8408 %}
8409
8410 // Manifest a CmpUL result in an integer register.
8411 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8412 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8413 %{
8414 match(Set dst (CmpUL3 src1 src2));
8415 effect(KILL flags);
8416
8417 ins_cost(INSN_COST * 3);
8418 format %{
8419 "cmp $src1, $src2\n\t"
8420 "csetw $dst, ne\n\t"
8421 "cnegw $dst, lo\t# CmpUL3(reg)"
8422 %}
8423 ins_encode %{
8424 __ cmp($src1$$Register, $src2$$Register);
8425 __ csetw($dst$$Register, Assembler::NE);
8426 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8427 %}
8428
8429 ins_pipe(pipe_class_default);
8430 %}
8431
8432 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8433 %{
8434 match(Set dst (CmpUL3 src1 src2));
8435 effect(KILL flags);
8436
8437 ins_cost(INSN_COST * 3);
8438 format %{
8439 "subs zr, $src1, $src2\n\t"
8440 "csetw $dst, ne\n\t"
8441 "cnegw $dst, lo\t# CmpUL3(imm)"
8442 %}
8443 ins_encode %{
8444 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8445 __ csetw($dst$$Register, Assembler::NE);
8446 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8447 %}
8448
8449 ins_pipe(pipe_class_default);
8450 %}
8451
8452 // Manifest a CmpL result in an integer register.
8453 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8454 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8455 %{
8456 match(Set dst (CmpL3 src1 src2));
8457 effect(KILL flags);
8458
8459 ins_cost(INSN_COST * 3);
8460 format %{
8461 "cmp $src1, $src2\n\t"
8462 "csetw $dst, ne\n\t"
8463 "cnegw $dst, lt\t# CmpL3(reg)"
8464 %}
8465 ins_encode %{
8466 __ cmp($src1$$Register, $src2$$Register);
8467 __ csetw($dst$$Register, Assembler::NE);
8468 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8469 %}
8470
8471 ins_pipe(pipe_class_default);
8472 %}
8473
8474 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8475 %{
8476 match(Set dst (CmpL3 src1 src2));
8477 effect(KILL flags);
8478
8479 ins_cost(INSN_COST * 3);
8480 format %{
8481 "subs zr, $src1, $src2\n\t"
8482 "csetw $dst, ne\n\t"
8483 "cnegw $dst, lt\t# CmpL3(imm)"
8484 %}
8485 ins_encode %{
8486 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8487 __ csetw($dst$$Register, Assembler::NE);
8488 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8489 %}
8490
8491 ins_pipe(pipe_class_default);
8492 %}
8493
8494 // ============================================================================
8495 // Conditional Move Instructions
8496
8497 // n.b. we have identical rules for both a signed compare op (cmpOp)
8498 // and an unsigned compare op (cmpOpU). it would be nice if we could
8499 // define an op class which merged both inputs and use it to type the
8500 // argument to a single rule. unfortunatelyt his fails because the
8501 // opclass does not live up to the COND_INTER interface of its
8502 // component operands. When the generic code tries to negate the
8503 // operand it ends up running the generci Machoper::negate method
8504 // which throws a ShouldNotHappen. So, we have to provide two flavours
8505 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8506
8507 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8508 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8509
8510 ins_cost(INSN_COST * 2);
8511 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8512
8513 ins_encode %{
8514 __ cselw(as_Register($dst$$reg),
8515 as_Register($src2$$reg),
8516 as_Register($src1$$reg),
8517 (Assembler::Condition)$cmp$$cmpcode);
8518 %}
8519
8520 ins_pipe(icond_reg_reg);
8521 %}
8522
8523 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8524 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8525
8526 ins_cost(INSN_COST * 2);
8527 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8528
8529 ins_encode %{
8530 __ cselw(as_Register($dst$$reg),
8531 as_Register($src2$$reg),
8532 as_Register($src1$$reg),
8533 (Assembler::Condition)$cmp$$cmpcode);
8534 %}
8535
8536 ins_pipe(icond_reg_reg);
8537 %}
8538
8539 // special cases where one arg is zero
8540
8541 // n.b. this is selected in preference to the rule above because it
8542 // avoids loading constant 0 into a source register
8543
8544 // TODO
8545 // we ought only to be able to cull one of these variants as the ideal
8546 // transforms ought always to order the zero consistently (to left/right?)
8547
8548 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8549 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8550
8551 ins_cost(INSN_COST * 2);
8552 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8553
8554 ins_encode %{
8555 __ cselw(as_Register($dst$$reg),
8556 as_Register($src$$reg),
8557 zr,
8558 (Assembler::Condition)$cmp$$cmpcode);
8559 %}
8560
8561 ins_pipe(icond_reg);
8562 %}
8563
8564 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8565 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8566
8567 ins_cost(INSN_COST * 2);
8568 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8569
8570 ins_encode %{
8571 __ cselw(as_Register($dst$$reg),
8572 as_Register($src$$reg),
8573 zr,
8574 (Assembler::Condition)$cmp$$cmpcode);
8575 %}
8576
8577 ins_pipe(icond_reg);
8578 %}
8579
8580 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8581 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8582
8583 ins_cost(INSN_COST * 2);
8584 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8585
8586 ins_encode %{
8587 __ cselw(as_Register($dst$$reg),
8588 zr,
8589 as_Register($src$$reg),
8590 (Assembler::Condition)$cmp$$cmpcode);
8591 %}
8592
8593 ins_pipe(icond_reg);
8594 %}
8595
8596 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8597 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8598
8599 ins_cost(INSN_COST * 2);
8600 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8601
8602 ins_encode %{
8603 __ cselw(as_Register($dst$$reg),
8604 zr,
8605 as_Register($src$$reg),
8606 (Assembler::Condition)$cmp$$cmpcode);
8607 %}
8608
8609 ins_pipe(icond_reg);
8610 %}
8611
8612 // special case for creating a boolean 0 or 1
8613
8614 // n.b. this is selected in preference to the rule above because it
8615 // avoids loading constants 0 and 1 into a source register
8616
8617 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8618 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8619
8620 ins_cost(INSN_COST * 2);
8621 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8622
8623 ins_encode %{
8624 // equivalently
8625 // cset(as_Register($dst$$reg),
8626 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8627 __ csincw(as_Register($dst$$reg),
8628 zr,
8629 zr,
8630 (Assembler::Condition)$cmp$$cmpcode);
8631 %}
8632
8633 ins_pipe(icond_none);
8634 %}
8635
8636 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8637 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8638
8639 ins_cost(INSN_COST * 2);
8640 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8641
8642 ins_encode %{
8643 // equivalently
8644 // cset(as_Register($dst$$reg),
8645 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8646 __ csincw(as_Register($dst$$reg),
8647 zr,
8648 zr,
8649 (Assembler::Condition)$cmp$$cmpcode);
8650 %}
8651
8652 ins_pipe(icond_none);
8653 %}
8654
8655 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8656 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8657
8658 ins_cost(INSN_COST * 2);
8659 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8660
8661 ins_encode %{
8662 __ csel(as_Register($dst$$reg),
8663 as_Register($src2$$reg),
8664 as_Register($src1$$reg),
8665 (Assembler::Condition)$cmp$$cmpcode);
8666 %}
8667
8668 ins_pipe(icond_reg_reg);
8669 %}
8670
8671 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8672 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8673
8674 ins_cost(INSN_COST * 2);
8675 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8676
8677 ins_encode %{
8678 __ csel(as_Register($dst$$reg),
8679 as_Register($src2$$reg),
8680 as_Register($src1$$reg),
8681 (Assembler::Condition)$cmp$$cmpcode);
8682 %}
8683
8684 ins_pipe(icond_reg_reg);
8685 %}
8686
8687 // special cases where one arg is zero
8688
8689 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8690 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8691
8692 ins_cost(INSN_COST * 2);
8693 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8694
8695 ins_encode %{
8696 __ csel(as_Register($dst$$reg),
8697 zr,
8698 as_Register($src$$reg),
8699 (Assembler::Condition)$cmp$$cmpcode);
8700 %}
8701
8702 ins_pipe(icond_reg);
8703 %}
8704
8705 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8706 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8707
8708 ins_cost(INSN_COST * 2);
8709 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8710
8711 ins_encode %{
8712 __ csel(as_Register($dst$$reg),
8713 zr,
8714 as_Register($src$$reg),
8715 (Assembler::Condition)$cmp$$cmpcode);
8716 %}
8717
8718 ins_pipe(icond_reg);
8719 %}
8720
8721 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8722 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8723
8724 ins_cost(INSN_COST * 2);
8725 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8726
8727 ins_encode %{
8728 __ csel(as_Register($dst$$reg),
8729 as_Register($src$$reg),
8730 zr,
8731 (Assembler::Condition)$cmp$$cmpcode);
8732 %}
8733
8734 ins_pipe(icond_reg);
8735 %}
8736
8737 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8738 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8739
8740 ins_cost(INSN_COST * 2);
8741 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8742
8743 ins_encode %{
8744 __ csel(as_Register($dst$$reg),
8745 as_Register($src$$reg),
8746 zr,
8747 (Assembler::Condition)$cmp$$cmpcode);
8748 %}
8749
8750 ins_pipe(icond_reg);
8751 %}
8752
8753 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8754 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8755
8756 ins_cost(INSN_COST * 2);
8757 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8758
8759 ins_encode %{
8760 __ csel(as_Register($dst$$reg),
8761 as_Register($src2$$reg),
8762 as_Register($src1$$reg),
8763 (Assembler::Condition)$cmp$$cmpcode);
8764 %}
8765
8766 ins_pipe(icond_reg_reg);
8767 %}
8768
8769 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8770 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8771
8772 ins_cost(INSN_COST * 2);
8773 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8774
8775 ins_encode %{
8776 __ csel(as_Register($dst$$reg),
8777 as_Register($src2$$reg),
8778 as_Register($src1$$reg),
8779 (Assembler::Condition)$cmp$$cmpcode);
8780 %}
8781
8782 ins_pipe(icond_reg_reg);
8783 %}
8784
8785 // special cases where one arg is zero
8786
8787 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8788 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8789
8790 ins_cost(INSN_COST * 2);
8791 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8792
8793 ins_encode %{
8794 __ csel(as_Register($dst$$reg),
8795 zr,
8796 as_Register($src$$reg),
8797 (Assembler::Condition)$cmp$$cmpcode);
8798 %}
8799
8800 ins_pipe(icond_reg);
8801 %}
8802
8803 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8804 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8805
8806 ins_cost(INSN_COST * 2);
8807 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8808
8809 ins_encode %{
8810 __ csel(as_Register($dst$$reg),
8811 zr,
8812 as_Register($src$$reg),
8813 (Assembler::Condition)$cmp$$cmpcode);
8814 %}
8815
8816 ins_pipe(icond_reg);
8817 %}
8818
8819 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8820 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8821
8822 ins_cost(INSN_COST * 2);
8823 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8824
8825 ins_encode %{
8826 __ csel(as_Register($dst$$reg),
8827 as_Register($src$$reg),
8828 zr,
8829 (Assembler::Condition)$cmp$$cmpcode);
8830 %}
8831
8832 ins_pipe(icond_reg);
8833 %}
8834
8835 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8836 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8837
8838 ins_cost(INSN_COST * 2);
8839 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8840
8841 ins_encode %{
8842 __ csel(as_Register($dst$$reg),
8843 as_Register($src$$reg),
8844 zr,
8845 (Assembler::Condition)$cmp$$cmpcode);
8846 %}
8847
8848 ins_pipe(icond_reg);
8849 %}
8850
8851 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8852 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8853
8854 ins_cost(INSN_COST * 2);
8855 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8856
8857 ins_encode %{
8858 __ cselw(as_Register($dst$$reg),
8859 as_Register($src2$$reg),
8860 as_Register($src1$$reg),
8861 (Assembler::Condition)$cmp$$cmpcode);
8862 %}
8863
8864 ins_pipe(icond_reg_reg);
8865 %}
8866
8867 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8868 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8869
8870 ins_cost(INSN_COST * 2);
8871 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8872
8873 ins_encode %{
8874 __ cselw(as_Register($dst$$reg),
8875 as_Register($src2$$reg),
8876 as_Register($src1$$reg),
8877 (Assembler::Condition)$cmp$$cmpcode);
8878 %}
8879
8880 ins_pipe(icond_reg_reg);
8881 %}
8882
8883 // special cases where one arg is zero
8884
8885 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8886 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8887
8888 ins_cost(INSN_COST * 2);
8889 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8890
8891 ins_encode %{
8892 __ cselw(as_Register($dst$$reg),
8893 zr,
8894 as_Register($src$$reg),
8895 (Assembler::Condition)$cmp$$cmpcode);
8896 %}
8897
8898 ins_pipe(icond_reg);
8899 %}
8900
8901 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8902 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8903
8904 ins_cost(INSN_COST * 2);
8905 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8906
8907 ins_encode %{
8908 __ cselw(as_Register($dst$$reg),
8909 zr,
8910 as_Register($src$$reg),
8911 (Assembler::Condition)$cmp$$cmpcode);
8912 %}
8913
8914 ins_pipe(icond_reg);
8915 %}
8916
8917 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8918 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8919
8920 ins_cost(INSN_COST * 2);
8921 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8922
8923 ins_encode %{
8924 __ cselw(as_Register($dst$$reg),
8925 as_Register($src$$reg),
8926 zr,
8927 (Assembler::Condition)$cmp$$cmpcode);
8928 %}
8929
8930 ins_pipe(icond_reg);
8931 %}
8932
8933 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8934 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8935
8936 ins_cost(INSN_COST * 2);
8937 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8938
8939 ins_encode %{
8940 __ cselw(as_Register($dst$$reg),
8941 as_Register($src$$reg),
8942 zr,
8943 (Assembler::Condition)$cmp$$cmpcode);
8944 %}
8945
8946 ins_pipe(icond_reg);
8947 %}
8948
8949 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8950 %{
8951 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8952
8953 ins_cost(INSN_COST * 3);
8954
8955 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8956 ins_encode %{
8957 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8958 __ fcsels(as_FloatRegister($dst$$reg),
8959 as_FloatRegister($src2$$reg),
8960 as_FloatRegister($src1$$reg),
8961 cond);
8962 %}
8963
8964 ins_pipe(fp_cond_reg_reg_s);
8965 %}
8966
8967 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8968 %{
8969 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8970
8971 ins_cost(INSN_COST * 3);
8972
8973 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8974 ins_encode %{
8975 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8976 __ fcsels(as_FloatRegister($dst$$reg),
8977 as_FloatRegister($src2$$reg),
8978 as_FloatRegister($src1$$reg),
8979 cond);
8980 %}
8981
8982 ins_pipe(fp_cond_reg_reg_s);
8983 %}
8984
8985 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8986 %{
8987 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8988
8989 ins_cost(INSN_COST * 3);
8990
8991 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8992 ins_encode %{
8993 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8994 __ fcseld(as_FloatRegister($dst$$reg),
8995 as_FloatRegister($src2$$reg),
8996 as_FloatRegister($src1$$reg),
8997 cond);
8998 %}
8999
9000 ins_pipe(fp_cond_reg_reg_d);
9001 %}
9002
9003 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9004 %{
9005 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9006
9007 ins_cost(INSN_COST * 3);
9008
9009 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9010 ins_encode %{
9011 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9012 __ fcseld(as_FloatRegister($dst$$reg),
9013 as_FloatRegister($src2$$reg),
9014 as_FloatRegister($src1$$reg),
9015 cond);
9016 %}
9017
9018 ins_pipe(fp_cond_reg_reg_d);
9019 %}
9020
9021 // ============================================================================
9022 // Arithmetic Instructions
9023 //
9024
9025 // Integer Addition
9026
9027 // TODO
9028 // these currently employ operations which do not set CR and hence are
9029 // not flagged as killing CR but we would like to isolate the cases
9030 // where we want to set flags from those where we don't. need to work
9031 // out how to do that.
9032
9033 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9034 match(Set dst (AddI src1 src2));
9035
9036 ins_cost(INSN_COST);
9037 format %{ "addw $dst, $src1, $src2" %}
9038
9039 ins_encode %{
9040 __ addw(as_Register($dst$$reg),
9041 as_Register($src1$$reg),
9042 as_Register($src2$$reg));
9043 %}
9044
9045 ins_pipe(ialu_reg_reg);
9046 %}
9047
9048 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9049 match(Set dst (AddI src1 src2));
9050
9051 ins_cost(INSN_COST);
9052 format %{ "addw $dst, $src1, $src2" %}
9053
9054 // use opcode to indicate that this is an add not a sub
9055 opcode(0x0);
9056
9057 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9058
9059 ins_pipe(ialu_reg_imm);
9060 %}
9061
9062 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9063 match(Set dst (AddI (ConvL2I src1) src2));
9064
9065 ins_cost(INSN_COST);
9066 format %{ "addw $dst, $src1, $src2" %}
9067
9068 // use opcode to indicate that this is an add not a sub
9069 opcode(0x0);
9070
9071 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9072
9073 ins_pipe(ialu_reg_imm);
9074 %}
9075
9076 // Pointer Addition
9077 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9078 match(Set dst (AddP src1 src2));
9079
9080 ins_cost(INSN_COST);
9081 format %{ "add $dst, $src1, $src2\t# ptr" %}
9082
9083 ins_encode %{
9084 __ add(as_Register($dst$$reg),
9085 as_Register($src1$$reg),
9086 as_Register($src2$$reg));
9087 %}
9088
9089 ins_pipe(ialu_reg_reg);
9090 %}
9091
9092 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9093 match(Set dst (AddP src1 (ConvI2L src2)));
9094
9095 ins_cost(1.9 * INSN_COST);
9096 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9097
9098 ins_encode %{
9099 __ add(as_Register($dst$$reg),
9100 as_Register($src1$$reg),
9101 as_Register($src2$$reg), ext::sxtw);
9102 %}
9103
9104 ins_pipe(ialu_reg_reg);
9105 %}
9106
9107 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9108 match(Set dst (AddP src1 (LShiftL src2 scale)));
9109
9110 ins_cost(1.9 * INSN_COST);
9111 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9112
9113 ins_encode %{
9114 __ lea(as_Register($dst$$reg),
9115 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9116 Address::lsl($scale$$constant)));
9117 %}
9118
9119 ins_pipe(ialu_reg_reg_shift);
9120 %}
9121
9122 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9123 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9124
9125 ins_cost(1.9 * INSN_COST);
9126 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9127
9128 ins_encode %{
9129 __ lea(as_Register($dst$$reg),
9130 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9131 Address::sxtw($scale$$constant)));
9132 %}
9133
9134 ins_pipe(ialu_reg_reg_shift);
9135 %}
9136
9137 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9138 match(Set dst (LShiftL (ConvI2L src) scale));
9139
9140 ins_cost(INSN_COST);
9141 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9142
9143 ins_encode %{
9144 __ sbfiz(as_Register($dst$$reg),
9145 as_Register($src$$reg),
9146 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9147 %}
9148
9149 ins_pipe(ialu_reg_shift);
9150 %}
9151
9152 // Pointer Immediate Addition
9153 // n.b. this needs to be more expensive than using an indirect memory
9154 // operand
9155 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9156 match(Set dst (AddP src1 src2));
9157
9158 ins_cost(INSN_COST);
9159 format %{ "add $dst, $src1, $src2\t# ptr" %}
9160
9161 // use opcode to indicate that this is an add not a sub
9162 opcode(0x0);
9163
9164 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9165
9166 ins_pipe(ialu_reg_imm);
9167 %}
9168
9169 // Long Addition
9170 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9171
9172 match(Set dst (AddL src1 src2));
9173
9174 ins_cost(INSN_COST);
9175 format %{ "add $dst, $src1, $src2" %}
9176
9177 ins_encode %{
9178 __ add(as_Register($dst$$reg),
9179 as_Register($src1$$reg),
9180 as_Register($src2$$reg));
9181 %}
9182
9183 ins_pipe(ialu_reg_reg);
9184 %}
9185
9186 // No constant pool entries requiredLong Immediate Addition.
9187 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9188 match(Set dst (AddL src1 src2));
9189
9190 ins_cost(INSN_COST);
9191 format %{ "add $dst, $src1, $src2" %}
9192
9193 // use opcode to indicate that this is an add not a sub
9194 opcode(0x0);
9195
9196 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9197
9198 ins_pipe(ialu_reg_imm);
9199 %}
9200
9201 // Integer Subtraction
9202 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9203 match(Set dst (SubI src1 src2));
9204
9205 ins_cost(INSN_COST);
9206 format %{ "subw $dst, $src1, $src2" %}
9207
9208 ins_encode %{
9209 __ subw(as_Register($dst$$reg),
9210 as_Register($src1$$reg),
9211 as_Register($src2$$reg));
9212 %}
9213
9214 ins_pipe(ialu_reg_reg);
9215 %}
9216
9217 // Immediate Subtraction
9218 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9219 match(Set dst (SubI src1 src2));
9220
9221 ins_cost(INSN_COST);
9222 format %{ "subw $dst, $src1, $src2" %}
9223
9224 // use opcode to indicate that this is a sub not an add
9225 opcode(0x1);
9226
9227 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9228
9229 ins_pipe(ialu_reg_imm);
9230 %}
9231
9232 // Long Subtraction
9233 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9234
9235 match(Set dst (SubL src1 src2));
9236
9237 ins_cost(INSN_COST);
9238 format %{ "sub $dst, $src1, $src2" %}
9239
9240 ins_encode %{
9241 __ sub(as_Register($dst$$reg),
9242 as_Register($src1$$reg),
9243 as_Register($src2$$reg));
9244 %}
9245
9246 ins_pipe(ialu_reg_reg);
9247 %}
9248
9249 // No constant pool entries requiredLong Immediate Subtraction.
9250 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9251 match(Set dst (SubL src1 src2));
9252
9253 ins_cost(INSN_COST);
9254 format %{ "sub$dst, $src1, $src2" %}
9255
9256 // use opcode to indicate that this is a sub not an add
9257 opcode(0x1);
9258
9259 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9260
9261 ins_pipe(ialu_reg_imm);
9262 %}
9263
9264 // Integer Negation (special case for sub)
9265
9266 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9267 match(Set dst (SubI zero src));
9268
9269 ins_cost(INSN_COST);
9270 format %{ "negw $dst, $src\t# int" %}
9271
9272 ins_encode %{
9273 __ negw(as_Register($dst$$reg),
9274 as_Register($src$$reg));
9275 %}
9276
9277 ins_pipe(ialu_reg);
9278 %}
9279
9280 // Long Negation
9281
9282 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9283 match(Set dst (SubL zero src));
9284
9285 ins_cost(INSN_COST);
9286 format %{ "neg $dst, $src\t# long" %}
9287
9288 ins_encode %{
9289 __ neg(as_Register($dst$$reg),
9290 as_Register($src$$reg));
9291 %}
9292
9293 ins_pipe(ialu_reg);
9294 %}
9295
9296 // Integer Multiply
9297
9298 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9299 match(Set dst (MulI src1 src2));
9300
9301 ins_cost(INSN_COST * 3);
9302 format %{ "mulw $dst, $src1, $src2" %}
9303
9304 ins_encode %{
9305 __ mulw(as_Register($dst$$reg),
9306 as_Register($src1$$reg),
9307 as_Register($src2$$reg));
9308 %}
9309
9310 ins_pipe(imul_reg_reg);
9311 %}
9312
9313 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9314 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9315
9316 ins_cost(INSN_COST * 3);
9317 format %{ "smull $dst, $src1, $src2" %}
9318
9319 ins_encode %{
9320 __ smull(as_Register($dst$$reg),
9321 as_Register($src1$$reg),
9322 as_Register($src2$$reg));
9323 %}
9324
9325 ins_pipe(imul_reg_reg);
9326 %}
9327
9328 // Long Multiply
9329
9330 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9331 match(Set dst (MulL src1 src2));
9332
9333 ins_cost(INSN_COST * 5);
9334 format %{ "mul $dst, $src1, $src2" %}
9335
9336 ins_encode %{
9337 __ mul(as_Register($dst$$reg),
9338 as_Register($src1$$reg),
9339 as_Register($src2$$reg));
9340 %}
9341
9342 ins_pipe(lmul_reg_reg);
9343 %}
9344
9345 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9346 %{
9347 match(Set dst (MulHiL src1 src2));
9348
9349 ins_cost(INSN_COST * 7);
9350 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9351
9352 ins_encode %{
9353 __ smulh(as_Register($dst$$reg),
9354 as_Register($src1$$reg),
9355 as_Register($src2$$reg));
9356 %}
9357
9358 ins_pipe(lmul_reg_reg);
9359 %}
9360
9361 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9362 %{
9363 match(Set dst (UMulHiL src1 src2));
9364
9365 ins_cost(INSN_COST * 7);
9366 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9367
9368 ins_encode %{
9369 __ umulh(as_Register($dst$$reg),
9370 as_Register($src1$$reg),
9371 as_Register($src2$$reg));
9372 %}
9373
9374 ins_pipe(lmul_reg_reg);
9375 %}
9376
9377 // Combined Integer Multiply & Add/Sub
9378
9379 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9380 match(Set dst (AddI src3 (MulI src1 src2)));
9381
9382 ins_cost(INSN_COST * 3);
9383 format %{ "madd $dst, $src1, $src2, $src3" %}
9384
9385 ins_encode %{
9386 __ maddw(as_Register($dst$$reg),
9387 as_Register($src1$$reg),
9388 as_Register($src2$$reg),
9389 as_Register($src3$$reg));
9390 %}
9391
9392 ins_pipe(imac_reg_reg);
9393 %}
9394
9395 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9396 match(Set dst (SubI src3 (MulI src1 src2)));
9397
9398 ins_cost(INSN_COST * 3);
9399 format %{ "msub $dst, $src1, $src2, $src3" %}
9400
9401 ins_encode %{
9402 __ msubw(as_Register($dst$$reg),
9403 as_Register($src1$$reg),
9404 as_Register($src2$$reg),
9405 as_Register($src3$$reg));
9406 %}
9407
9408 ins_pipe(imac_reg_reg);
9409 %}
9410
9411 // Combined Integer Multiply & Neg
9412
9413 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9414 match(Set dst (MulI (SubI zero src1) src2));
9415
9416 ins_cost(INSN_COST * 3);
9417 format %{ "mneg $dst, $src1, $src2" %}
9418
9419 ins_encode %{
9420 __ mnegw(as_Register($dst$$reg),
9421 as_Register($src1$$reg),
9422 as_Register($src2$$reg));
9423 %}
9424
9425 ins_pipe(imac_reg_reg);
9426 %}
9427
9428 // Combined Long Multiply & Add/Sub
9429
9430 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9431 match(Set dst (AddL src3 (MulL src1 src2)));
9432
9433 ins_cost(INSN_COST * 5);
9434 format %{ "madd $dst, $src1, $src2, $src3" %}
9435
9436 ins_encode %{
9437 __ madd(as_Register($dst$$reg),
9438 as_Register($src1$$reg),
9439 as_Register($src2$$reg),
9440 as_Register($src3$$reg));
9441 %}
9442
9443 ins_pipe(lmac_reg_reg);
9444 %}
9445
9446 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9447 match(Set dst (SubL src3 (MulL src1 src2)));
9448
9449 ins_cost(INSN_COST * 5);
9450 format %{ "msub $dst, $src1, $src2, $src3" %}
9451
9452 ins_encode %{
9453 __ msub(as_Register($dst$$reg),
9454 as_Register($src1$$reg),
9455 as_Register($src2$$reg),
9456 as_Register($src3$$reg));
9457 %}
9458
9459 ins_pipe(lmac_reg_reg);
9460 %}
9461
9462 // Combined Long Multiply & Neg
9463
9464 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9465 match(Set dst (MulL (SubL zero src1) src2));
9466
9467 ins_cost(INSN_COST * 5);
9468 format %{ "mneg $dst, $src1, $src2" %}
9469
9470 ins_encode %{
9471 __ mneg(as_Register($dst$$reg),
9472 as_Register($src1$$reg),
9473 as_Register($src2$$reg));
9474 %}
9475
9476 ins_pipe(lmac_reg_reg);
9477 %}
9478
9479 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9480
9481 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9482 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9483
9484 ins_cost(INSN_COST * 3);
9485 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9486
9487 ins_encode %{
9488 __ smaddl(as_Register($dst$$reg),
9489 as_Register($src1$$reg),
9490 as_Register($src2$$reg),
9491 as_Register($src3$$reg));
9492 %}
9493
9494 ins_pipe(imac_reg_reg);
9495 %}
9496
9497 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9498 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9499
9500 ins_cost(INSN_COST * 3);
9501 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9502
9503 ins_encode %{
9504 __ smsubl(as_Register($dst$$reg),
9505 as_Register($src1$$reg),
9506 as_Register($src2$$reg),
9507 as_Register($src3$$reg));
9508 %}
9509
9510 ins_pipe(imac_reg_reg);
9511 %}
9512
9513 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9514 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9515
9516 ins_cost(INSN_COST * 3);
9517 format %{ "smnegl $dst, $src1, $src2" %}
9518
9519 ins_encode %{
9520 __ smnegl(as_Register($dst$$reg),
9521 as_Register($src1$$reg),
9522 as_Register($src2$$reg));
9523 %}
9524
9525 ins_pipe(imac_reg_reg);
9526 %}
9527
9528 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9529
9530 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9531 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9532
9533 ins_cost(INSN_COST * 5);
9534 format %{ "mulw rscratch1, $src1, $src2\n\t"
9535 "maddw $dst, $src3, $src4, rscratch1" %}
9536
9537 ins_encode %{
9538 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9539 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9540
9541 ins_pipe(imac_reg_reg);
9542 %}
9543
9544 // Integer Divide
9545
9546 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9547 match(Set dst (DivI src1 src2));
9548
9549 ins_cost(INSN_COST * 19);
9550 format %{ "sdivw $dst, $src1, $src2" %}
9551
9552 ins_encode(aarch64_enc_divw(dst, src1, src2));
9553 ins_pipe(idiv_reg_reg);
9554 %}
9555
9556 // Long Divide
9557
9558 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9559 match(Set dst (DivL src1 src2));
9560
9561 ins_cost(INSN_COST * 35);
9562 format %{ "sdiv $dst, $src1, $src2" %}
9563
9564 ins_encode(aarch64_enc_div(dst, src1, src2));
9565 ins_pipe(ldiv_reg_reg);
9566 %}
9567
9568 // Integer Remainder
9569
9570 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9571 match(Set dst (ModI src1 src2));
9572
9573 ins_cost(INSN_COST * 22);
9574 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9575 "msubw $dst, rscratch1, $src2, $src1" %}
9576
9577 ins_encode(aarch64_enc_modw(dst, src1, src2));
9578 ins_pipe(idiv_reg_reg);
9579 %}
9580
9581 // Long Remainder
9582
9583 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9584 match(Set dst (ModL src1 src2));
9585
9586 ins_cost(INSN_COST * 38);
9587 format %{ "sdiv rscratch1, $src1, $src2\n"
9588 "msub $dst, rscratch1, $src2, $src1" %}
9589
9590 ins_encode(aarch64_enc_mod(dst, src1, src2));
9591 ins_pipe(ldiv_reg_reg);
9592 %}
9593
9594 // Unsigned Integer Divide
9595
9596 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9597 match(Set dst (UDivI src1 src2));
9598
9599 ins_cost(INSN_COST * 19);
9600 format %{ "udivw $dst, $src1, $src2" %}
9601
9602 ins_encode %{
9603 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9604 %}
9605
9606 ins_pipe(idiv_reg_reg);
9607 %}
9608
9609 // Unsigned Long Divide
9610
9611 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9612 match(Set dst (UDivL src1 src2));
9613
9614 ins_cost(INSN_COST * 35);
9615 format %{ "udiv $dst, $src1, $src2" %}
9616
9617 ins_encode %{
9618 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9619 %}
9620
9621 ins_pipe(ldiv_reg_reg);
9622 %}
9623
9624 // Unsigned Integer Remainder
9625
9626 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9627 match(Set dst (UModI src1 src2));
9628
9629 ins_cost(INSN_COST * 22);
9630 format %{ "udivw rscratch1, $src1, $src2\n\t"
9631 "msubw $dst, rscratch1, $src2, $src1" %}
9632
9633 ins_encode %{
9634 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9635 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9636 %}
9637
9638 ins_pipe(idiv_reg_reg);
9639 %}
9640
9641 // Unsigned Long Remainder
9642
9643 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9644 match(Set dst (UModL src1 src2));
9645
9646 ins_cost(INSN_COST * 38);
9647 format %{ "udiv rscratch1, $src1, $src2\n"
9648 "msub $dst, rscratch1, $src2, $src1" %}
9649
9650 ins_encode %{
9651 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9652 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9653 %}
9654
9655 ins_pipe(ldiv_reg_reg);
9656 %}
9657
9658 // Integer Shifts
9659
9660 // Shift Left Register
9661 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9662 match(Set dst (LShiftI src1 src2));
9663
9664 ins_cost(INSN_COST * 2);
9665 format %{ "lslvw $dst, $src1, $src2" %}
9666
9667 ins_encode %{
9668 __ lslvw(as_Register($dst$$reg),
9669 as_Register($src1$$reg),
9670 as_Register($src2$$reg));
9671 %}
9672
9673 ins_pipe(ialu_reg_reg_vshift);
9674 %}
9675
9676 // Shift Left Immediate
9677 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9678 match(Set dst (LShiftI src1 src2));
9679
9680 ins_cost(INSN_COST);
9681 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9682
9683 ins_encode %{
9684 __ lslw(as_Register($dst$$reg),
9685 as_Register($src1$$reg),
9686 $src2$$constant & 0x1f);
9687 %}
9688
9689 ins_pipe(ialu_reg_shift);
9690 %}
9691
9692 // Shift Right Logical Register
9693 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9694 match(Set dst (URShiftI src1 src2));
9695
9696 ins_cost(INSN_COST * 2);
9697 format %{ "lsrvw $dst, $src1, $src2" %}
9698
9699 ins_encode %{
9700 __ lsrvw(as_Register($dst$$reg),
9701 as_Register($src1$$reg),
9702 as_Register($src2$$reg));
9703 %}
9704
9705 ins_pipe(ialu_reg_reg_vshift);
9706 %}
9707
9708 // Shift Right Logical Immediate
9709 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9710 match(Set dst (URShiftI src1 src2));
9711
9712 ins_cost(INSN_COST);
9713 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9714
9715 ins_encode %{
9716 __ lsrw(as_Register($dst$$reg),
9717 as_Register($src1$$reg),
9718 $src2$$constant & 0x1f);
9719 %}
9720
9721 ins_pipe(ialu_reg_shift);
9722 %}
9723
9724 // Shift Right Arithmetic Register
9725 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9726 match(Set dst (RShiftI src1 src2));
9727
9728 ins_cost(INSN_COST * 2);
9729 format %{ "asrvw $dst, $src1, $src2" %}
9730
9731 ins_encode %{
9732 __ asrvw(as_Register($dst$$reg),
9733 as_Register($src1$$reg),
9734 as_Register($src2$$reg));
9735 %}
9736
9737 ins_pipe(ialu_reg_reg_vshift);
9738 %}
9739
9740 // Shift Right Arithmetic Immediate
9741 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9742 match(Set dst (RShiftI src1 src2));
9743
9744 ins_cost(INSN_COST);
9745 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9746
9747 ins_encode %{
9748 __ asrw(as_Register($dst$$reg),
9749 as_Register($src1$$reg),
9750 $src2$$constant & 0x1f);
9751 %}
9752
9753 ins_pipe(ialu_reg_shift);
9754 %}
9755
9756 // Combined Int Mask and Right Shift (using UBFM)
9757 // TODO
9758
9759 // Long Shifts
9760
9761 // Shift Left Register
9762 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9763 match(Set dst (LShiftL src1 src2));
9764
9765 ins_cost(INSN_COST * 2);
9766 format %{ "lslv $dst, $src1, $src2" %}
9767
9768 ins_encode %{
9769 __ lslv(as_Register($dst$$reg),
9770 as_Register($src1$$reg),
9771 as_Register($src2$$reg));
9772 %}
9773
9774 ins_pipe(ialu_reg_reg_vshift);
9775 %}
9776
9777 // Shift Left Immediate
9778 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9779 match(Set dst (LShiftL src1 src2));
9780
9781 ins_cost(INSN_COST);
9782 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9783
9784 ins_encode %{
9785 __ lsl(as_Register($dst$$reg),
9786 as_Register($src1$$reg),
9787 $src2$$constant & 0x3f);
9788 %}
9789
9790 ins_pipe(ialu_reg_shift);
9791 %}
9792
9793 // Shift Right Logical Register
9794 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9795 match(Set dst (URShiftL src1 src2));
9796
9797 ins_cost(INSN_COST * 2);
9798 format %{ "lsrv $dst, $src1, $src2" %}
9799
9800 ins_encode %{
9801 __ lsrv(as_Register($dst$$reg),
9802 as_Register($src1$$reg),
9803 as_Register($src2$$reg));
9804 %}
9805
9806 ins_pipe(ialu_reg_reg_vshift);
9807 %}
9808
9809 // Shift Right Logical Immediate
9810 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9811 match(Set dst (URShiftL src1 src2));
9812
9813 ins_cost(INSN_COST);
9814 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9815
9816 ins_encode %{
9817 __ lsr(as_Register($dst$$reg),
9818 as_Register($src1$$reg),
9819 $src2$$constant & 0x3f);
9820 %}
9821
9822 ins_pipe(ialu_reg_shift);
9823 %}
9824
9825 // A special-case pattern for card table stores.
9826 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9827 match(Set dst (URShiftL (CastP2X src1) src2));
9828
9829 ins_cost(INSN_COST);
9830 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9831
9832 ins_encode %{
9833 __ lsr(as_Register($dst$$reg),
9834 as_Register($src1$$reg),
9835 $src2$$constant & 0x3f);
9836 %}
9837
9838 ins_pipe(ialu_reg_shift);
9839 %}
9840
9841 // Shift Right Arithmetic Register
9842 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9843 match(Set dst (RShiftL src1 src2));
9844
9845 ins_cost(INSN_COST * 2);
9846 format %{ "asrv $dst, $src1, $src2" %}
9847
9848 ins_encode %{
9849 __ asrv(as_Register($dst$$reg),
9850 as_Register($src1$$reg),
9851 as_Register($src2$$reg));
9852 %}
9853
9854 ins_pipe(ialu_reg_reg_vshift);
9855 %}
9856
9857 // Shift Right Arithmetic Immediate
9858 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9859 match(Set dst (RShiftL src1 src2));
9860
9861 ins_cost(INSN_COST);
9862 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9863
9864 ins_encode %{
9865 __ asr(as_Register($dst$$reg),
9866 as_Register($src1$$reg),
9867 $src2$$constant & 0x3f);
9868 %}
9869
9870 ins_pipe(ialu_reg_shift);
9871 %}
9872
9873 // BEGIN This section of the file is automatically generated. Do not edit --------------
9874 // This section is generated from aarch64_ad.m4
9875
9876 // This pattern is automatically generated from aarch64_ad.m4
9877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9878 instruct regL_not_reg(iRegLNoSp dst,
9879 iRegL src1, immL_M1 m1,
9880 rFlagsReg cr) %{
9881 match(Set dst (XorL src1 m1));
9882 ins_cost(INSN_COST);
9883 format %{ "eon $dst, $src1, zr" %}
9884
9885 ins_encode %{
9886 __ eon(as_Register($dst$$reg),
9887 as_Register($src1$$reg),
9888 zr,
9889 Assembler::LSL, 0);
9890 %}
9891
9892 ins_pipe(ialu_reg);
9893 %}
9894
9895 // This pattern is automatically generated from aarch64_ad.m4
9896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9897 instruct regI_not_reg(iRegINoSp dst,
9898 iRegIorL2I src1, immI_M1 m1,
9899 rFlagsReg cr) %{
9900 match(Set dst (XorI src1 m1));
9901 ins_cost(INSN_COST);
9902 format %{ "eonw $dst, $src1, zr" %}
9903
9904 ins_encode %{
9905 __ eonw(as_Register($dst$$reg),
9906 as_Register($src1$$reg),
9907 zr,
9908 Assembler::LSL, 0);
9909 %}
9910
9911 ins_pipe(ialu_reg);
9912 %}
9913
9914 // This pattern is automatically generated from aarch64_ad.m4
9915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9916 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9917 immI0 zero, iRegIorL2I src1, immI src2) %{
9918 match(Set dst (SubI zero (URShiftI src1 src2)));
9919
9920 ins_cost(1.9 * INSN_COST);
9921 format %{ "negw $dst, $src1, LSR $src2" %}
9922
9923 ins_encode %{
9924 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9925 Assembler::LSR, $src2$$constant & 0x1f);
9926 %}
9927
9928 ins_pipe(ialu_reg_shift);
9929 %}
9930
9931 // This pattern is automatically generated from aarch64_ad.m4
9932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9933 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9934 immI0 zero, iRegIorL2I src1, immI src2) %{
9935 match(Set dst (SubI zero (RShiftI src1 src2)));
9936
9937 ins_cost(1.9 * INSN_COST);
9938 format %{ "negw $dst, $src1, ASR $src2" %}
9939
9940 ins_encode %{
9941 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9942 Assembler::ASR, $src2$$constant & 0x1f);
9943 %}
9944
9945 ins_pipe(ialu_reg_shift);
9946 %}
9947
9948 // This pattern is automatically generated from aarch64_ad.m4
9949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9950 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9951 immI0 zero, iRegIorL2I src1, immI src2) %{
9952 match(Set dst (SubI zero (LShiftI src1 src2)));
9953
9954 ins_cost(1.9 * INSN_COST);
9955 format %{ "negw $dst, $src1, LSL $src2" %}
9956
9957 ins_encode %{
9958 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9959 Assembler::LSL, $src2$$constant & 0x1f);
9960 %}
9961
9962 ins_pipe(ialu_reg_shift);
9963 %}
9964
9965 // This pattern is automatically generated from aarch64_ad.m4
9966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9967 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9968 immL0 zero, iRegL src1, immI src2) %{
9969 match(Set dst (SubL zero (URShiftL src1 src2)));
9970
9971 ins_cost(1.9 * INSN_COST);
9972 format %{ "neg $dst, $src1, LSR $src2" %}
9973
9974 ins_encode %{
9975 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9976 Assembler::LSR, $src2$$constant & 0x3f);
9977 %}
9978
9979 ins_pipe(ialu_reg_shift);
9980 %}
9981
9982 // This pattern is automatically generated from aarch64_ad.m4
9983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9984 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9985 immL0 zero, iRegL src1, immI src2) %{
9986 match(Set dst (SubL zero (RShiftL src1 src2)));
9987
9988 ins_cost(1.9 * INSN_COST);
9989 format %{ "neg $dst, $src1, ASR $src2" %}
9990
9991 ins_encode %{
9992 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9993 Assembler::ASR, $src2$$constant & 0x3f);
9994 %}
9995
9996 ins_pipe(ialu_reg_shift);
9997 %}
9998
9999 // This pattern is automatically generated from aarch64_ad.m4
10000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10001 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10002 immL0 zero, iRegL src1, immI src2) %{
10003 match(Set dst (SubL zero (LShiftL src1 src2)));
10004
10005 ins_cost(1.9 * INSN_COST);
10006 format %{ "neg $dst, $src1, LSL $src2" %}
10007
10008 ins_encode %{
10009 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10010 Assembler::LSL, $src2$$constant & 0x3f);
10011 %}
10012
10013 ins_pipe(ialu_reg_shift);
10014 %}
10015
10016 // This pattern is automatically generated from aarch64_ad.m4
10017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10018 instruct AndI_reg_not_reg(iRegINoSp dst,
10019 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10020 match(Set dst (AndI src1 (XorI src2 m1)));
10021 ins_cost(INSN_COST);
10022 format %{ "bicw $dst, $src1, $src2" %}
10023
10024 ins_encode %{
10025 __ bicw(as_Register($dst$$reg),
10026 as_Register($src1$$reg),
10027 as_Register($src2$$reg),
10028 Assembler::LSL, 0);
10029 %}
10030
10031 ins_pipe(ialu_reg_reg);
10032 %}
10033
10034 // This pattern is automatically generated from aarch64_ad.m4
10035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10036 instruct AndL_reg_not_reg(iRegLNoSp dst,
10037 iRegL src1, iRegL src2, immL_M1 m1) %{
10038 match(Set dst (AndL src1 (XorL src2 m1)));
10039 ins_cost(INSN_COST);
10040 format %{ "bic $dst, $src1, $src2" %}
10041
10042 ins_encode %{
10043 __ bic(as_Register($dst$$reg),
10044 as_Register($src1$$reg),
10045 as_Register($src2$$reg),
10046 Assembler::LSL, 0);
10047 %}
10048
10049 ins_pipe(ialu_reg_reg);
10050 %}
10051
10052 // This pattern is automatically generated from aarch64_ad.m4
10053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10054 instruct OrI_reg_not_reg(iRegINoSp dst,
10055 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10056 match(Set dst (OrI src1 (XorI src2 m1)));
10057 ins_cost(INSN_COST);
10058 format %{ "ornw $dst, $src1, $src2" %}
10059
10060 ins_encode %{
10061 __ ornw(as_Register($dst$$reg),
10062 as_Register($src1$$reg),
10063 as_Register($src2$$reg),
10064 Assembler::LSL, 0);
10065 %}
10066
10067 ins_pipe(ialu_reg_reg);
10068 %}
10069
10070 // This pattern is automatically generated from aarch64_ad.m4
10071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10072 instruct OrL_reg_not_reg(iRegLNoSp dst,
10073 iRegL src1, iRegL src2, immL_M1 m1) %{
10074 match(Set dst (OrL src1 (XorL src2 m1)));
10075 ins_cost(INSN_COST);
10076 format %{ "orn $dst, $src1, $src2" %}
10077
10078 ins_encode %{
10079 __ orn(as_Register($dst$$reg),
10080 as_Register($src1$$reg),
10081 as_Register($src2$$reg),
10082 Assembler::LSL, 0);
10083 %}
10084
10085 ins_pipe(ialu_reg_reg);
10086 %}
10087
10088 // This pattern is automatically generated from aarch64_ad.m4
10089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10090 instruct XorI_reg_not_reg(iRegINoSp dst,
10091 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10092 match(Set dst (XorI m1 (XorI src2 src1)));
10093 ins_cost(INSN_COST);
10094 format %{ "eonw $dst, $src1, $src2" %}
10095
10096 ins_encode %{
10097 __ eonw(as_Register($dst$$reg),
10098 as_Register($src1$$reg),
10099 as_Register($src2$$reg),
10100 Assembler::LSL, 0);
10101 %}
10102
10103 ins_pipe(ialu_reg_reg);
10104 %}
10105
10106 // This pattern is automatically generated from aarch64_ad.m4
10107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10108 instruct XorL_reg_not_reg(iRegLNoSp dst,
10109 iRegL src1, iRegL src2, immL_M1 m1) %{
10110 match(Set dst (XorL m1 (XorL src2 src1)));
10111 ins_cost(INSN_COST);
10112 format %{ "eon $dst, $src1, $src2" %}
10113
10114 ins_encode %{
10115 __ eon(as_Register($dst$$reg),
10116 as_Register($src1$$reg),
10117 as_Register($src2$$reg),
10118 Assembler::LSL, 0);
10119 %}
10120
10121 ins_pipe(ialu_reg_reg);
10122 %}
10123
10124 // This pattern is automatically generated from aarch64_ad.m4
10125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10126 // val & (-1 ^ (val >>> shift)) ==> bicw
10127 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10128 iRegIorL2I src1, iRegIorL2I src2,
10129 immI src3, immI_M1 src4) %{
10130 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10131 ins_cost(1.9 * INSN_COST);
10132 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10133
10134 ins_encode %{
10135 __ bicw(as_Register($dst$$reg),
10136 as_Register($src1$$reg),
10137 as_Register($src2$$reg),
10138 Assembler::LSR,
10139 $src3$$constant & 0x1f);
10140 %}
10141
10142 ins_pipe(ialu_reg_reg_shift);
10143 %}
10144
10145 // This pattern is automatically generated from aarch64_ad.m4
10146 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10147 // val & (-1 ^ (val >>> shift)) ==> bic
10148 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10149 iRegL src1, iRegL src2,
10150 immI src3, immL_M1 src4) %{
10151 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10152 ins_cost(1.9 * INSN_COST);
10153 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10154
10155 ins_encode %{
10156 __ bic(as_Register($dst$$reg),
10157 as_Register($src1$$reg),
10158 as_Register($src2$$reg),
10159 Assembler::LSR,
10160 $src3$$constant & 0x3f);
10161 %}
10162
10163 ins_pipe(ialu_reg_reg_shift);
10164 %}
10165
10166 // This pattern is automatically generated from aarch64_ad.m4
10167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10168 // val & (-1 ^ (val >> shift)) ==> bicw
10169 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10170 iRegIorL2I src1, iRegIorL2I src2,
10171 immI src3, immI_M1 src4) %{
10172 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10173 ins_cost(1.9 * INSN_COST);
10174 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10175
10176 ins_encode %{
10177 __ bicw(as_Register($dst$$reg),
10178 as_Register($src1$$reg),
10179 as_Register($src2$$reg),
10180 Assembler::ASR,
10181 $src3$$constant & 0x1f);
10182 %}
10183
10184 ins_pipe(ialu_reg_reg_shift);
10185 %}
10186
10187 // This pattern is automatically generated from aarch64_ad.m4
10188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10189 // val & (-1 ^ (val >> shift)) ==> bic
10190 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10191 iRegL src1, iRegL src2,
10192 immI src3, immL_M1 src4) %{
10193 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10194 ins_cost(1.9 * INSN_COST);
10195 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10196
10197 ins_encode %{
10198 __ bic(as_Register($dst$$reg),
10199 as_Register($src1$$reg),
10200 as_Register($src2$$reg),
10201 Assembler::ASR,
10202 $src3$$constant & 0x3f);
10203 %}
10204
10205 ins_pipe(ialu_reg_reg_shift);
10206 %}
10207
10208 // This pattern is automatically generated from aarch64_ad.m4
10209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10210 // val & (-1 ^ (val ror shift)) ==> bicw
10211 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10212 iRegIorL2I src1, iRegIorL2I src2,
10213 immI src3, immI_M1 src4) %{
10214 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10215 ins_cost(1.9 * INSN_COST);
10216 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10217
10218 ins_encode %{
10219 __ bicw(as_Register($dst$$reg),
10220 as_Register($src1$$reg),
10221 as_Register($src2$$reg),
10222 Assembler::ROR,
10223 $src3$$constant & 0x1f);
10224 %}
10225
10226 ins_pipe(ialu_reg_reg_shift);
10227 %}
10228
10229 // This pattern is automatically generated from aarch64_ad.m4
10230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10231 // val & (-1 ^ (val ror shift)) ==> bic
10232 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10233 iRegL src1, iRegL src2,
10234 immI src3, immL_M1 src4) %{
10235 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10236 ins_cost(1.9 * INSN_COST);
10237 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10238
10239 ins_encode %{
10240 __ bic(as_Register($dst$$reg),
10241 as_Register($src1$$reg),
10242 as_Register($src2$$reg),
10243 Assembler::ROR,
10244 $src3$$constant & 0x3f);
10245 %}
10246
10247 ins_pipe(ialu_reg_reg_shift);
10248 %}
10249
10250 // This pattern is automatically generated from aarch64_ad.m4
10251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10252 // val & (-1 ^ (val << shift)) ==> bicw
10253 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10254 iRegIorL2I src1, iRegIorL2I src2,
10255 immI src3, immI_M1 src4) %{
10256 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10257 ins_cost(1.9 * INSN_COST);
10258 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10259
10260 ins_encode %{
10261 __ bicw(as_Register($dst$$reg),
10262 as_Register($src1$$reg),
10263 as_Register($src2$$reg),
10264 Assembler::LSL,
10265 $src3$$constant & 0x1f);
10266 %}
10267
10268 ins_pipe(ialu_reg_reg_shift);
10269 %}
10270
10271 // This pattern is automatically generated from aarch64_ad.m4
10272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10273 // val & (-1 ^ (val << shift)) ==> bic
10274 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10275 iRegL src1, iRegL src2,
10276 immI src3, immL_M1 src4) %{
10277 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10278 ins_cost(1.9 * INSN_COST);
10279 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10280
10281 ins_encode %{
10282 __ bic(as_Register($dst$$reg),
10283 as_Register($src1$$reg),
10284 as_Register($src2$$reg),
10285 Assembler::LSL,
10286 $src3$$constant & 0x3f);
10287 %}
10288
10289 ins_pipe(ialu_reg_reg_shift);
10290 %}
10291
10292 // This pattern is automatically generated from aarch64_ad.m4
10293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10294 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10295 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10296 iRegIorL2I src1, iRegIorL2I src2,
10297 immI src3, immI_M1 src4) %{
10298 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10299 ins_cost(1.9 * INSN_COST);
10300 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10301
10302 ins_encode %{
10303 __ eonw(as_Register($dst$$reg),
10304 as_Register($src1$$reg),
10305 as_Register($src2$$reg),
10306 Assembler::LSR,
10307 $src3$$constant & 0x1f);
10308 %}
10309
10310 ins_pipe(ialu_reg_reg_shift);
10311 %}
10312
10313 // This pattern is automatically generated from aarch64_ad.m4
10314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10315 // val ^ (-1 ^ (val >>> shift)) ==> eon
10316 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10317 iRegL src1, iRegL src2,
10318 immI src3, immL_M1 src4) %{
10319 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10320 ins_cost(1.9 * INSN_COST);
10321 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10322
10323 ins_encode %{
10324 __ eon(as_Register($dst$$reg),
10325 as_Register($src1$$reg),
10326 as_Register($src2$$reg),
10327 Assembler::LSR,
10328 $src3$$constant & 0x3f);
10329 %}
10330
10331 ins_pipe(ialu_reg_reg_shift);
10332 %}
10333
10334 // This pattern is automatically generated from aarch64_ad.m4
10335 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10336 // val ^ (-1 ^ (val >> shift)) ==> eonw
10337 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10338 iRegIorL2I src1, iRegIorL2I src2,
10339 immI src3, immI_M1 src4) %{
10340 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10341 ins_cost(1.9 * INSN_COST);
10342 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10343
10344 ins_encode %{
10345 __ eonw(as_Register($dst$$reg),
10346 as_Register($src1$$reg),
10347 as_Register($src2$$reg),
10348 Assembler::ASR,
10349 $src3$$constant & 0x1f);
10350 %}
10351
10352 ins_pipe(ialu_reg_reg_shift);
10353 %}
10354
10355 // This pattern is automatically generated from aarch64_ad.m4
10356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10357 // val ^ (-1 ^ (val >> shift)) ==> eon
10358 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10359 iRegL src1, iRegL src2,
10360 immI src3, immL_M1 src4) %{
10361 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10362 ins_cost(1.9 * INSN_COST);
10363 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10364
10365 ins_encode %{
10366 __ eon(as_Register($dst$$reg),
10367 as_Register($src1$$reg),
10368 as_Register($src2$$reg),
10369 Assembler::ASR,
10370 $src3$$constant & 0x3f);
10371 %}
10372
10373 ins_pipe(ialu_reg_reg_shift);
10374 %}
10375
10376 // This pattern is automatically generated from aarch64_ad.m4
10377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10378 // val ^ (-1 ^ (val ror shift)) ==> eonw
10379 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10380 iRegIorL2I src1, iRegIorL2I src2,
10381 immI src3, immI_M1 src4) %{
10382 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10383 ins_cost(1.9 * INSN_COST);
10384 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10385
10386 ins_encode %{
10387 __ eonw(as_Register($dst$$reg),
10388 as_Register($src1$$reg),
10389 as_Register($src2$$reg),
10390 Assembler::ROR,
10391 $src3$$constant & 0x1f);
10392 %}
10393
10394 ins_pipe(ialu_reg_reg_shift);
10395 %}
10396
10397 // This pattern is automatically generated from aarch64_ad.m4
10398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10399 // val ^ (-1 ^ (val ror shift)) ==> eon
10400 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10401 iRegL src1, iRegL src2,
10402 immI src3, immL_M1 src4) %{
10403 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10404 ins_cost(1.9 * INSN_COST);
10405 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10406
10407 ins_encode %{
10408 __ eon(as_Register($dst$$reg),
10409 as_Register($src1$$reg),
10410 as_Register($src2$$reg),
10411 Assembler::ROR,
10412 $src3$$constant & 0x3f);
10413 %}
10414
10415 ins_pipe(ialu_reg_reg_shift);
10416 %}
10417
10418 // This pattern is automatically generated from aarch64_ad.m4
10419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10420 // val ^ (-1 ^ (val << shift)) ==> eonw
10421 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10422 iRegIorL2I src1, iRegIorL2I src2,
10423 immI src3, immI_M1 src4) %{
10424 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10425 ins_cost(1.9 * INSN_COST);
10426 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10427
10428 ins_encode %{
10429 __ eonw(as_Register($dst$$reg),
10430 as_Register($src1$$reg),
10431 as_Register($src2$$reg),
10432 Assembler::LSL,
10433 $src3$$constant & 0x1f);
10434 %}
10435
10436 ins_pipe(ialu_reg_reg_shift);
10437 %}
10438
10439 // This pattern is automatically generated from aarch64_ad.m4
10440 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10441 // val ^ (-1 ^ (val << shift)) ==> eon
10442 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10443 iRegL src1, iRegL src2,
10444 immI src3, immL_M1 src4) %{
10445 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10446 ins_cost(1.9 * INSN_COST);
10447 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10448
10449 ins_encode %{
10450 __ eon(as_Register($dst$$reg),
10451 as_Register($src1$$reg),
10452 as_Register($src2$$reg),
10453 Assembler::LSL,
10454 $src3$$constant & 0x3f);
10455 %}
10456
10457 ins_pipe(ialu_reg_reg_shift);
10458 %}
10459
10460 // This pattern is automatically generated from aarch64_ad.m4
10461 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10462 // val | (-1 ^ (val >>> shift)) ==> ornw
10463 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10464 iRegIorL2I src1, iRegIorL2I src2,
10465 immI src3, immI_M1 src4) %{
10466 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10467 ins_cost(1.9 * INSN_COST);
10468 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10469
10470 ins_encode %{
10471 __ ornw(as_Register($dst$$reg),
10472 as_Register($src1$$reg),
10473 as_Register($src2$$reg),
10474 Assembler::LSR,
10475 $src3$$constant & 0x1f);
10476 %}
10477
10478 ins_pipe(ialu_reg_reg_shift);
10479 %}
10480
10481 // This pattern is automatically generated from aarch64_ad.m4
10482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10483 // val | (-1 ^ (val >>> shift)) ==> orn
10484 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10485 iRegL src1, iRegL src2,
10486 immI src3, immL_M1 src4) %{
10487 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10488 ins_cost(1.9 * INSN_COST);
10489 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10490
10491 ins_encode %{
10492 __ orn(as_Register($dst$$reg),
10493 as_Register($src1$$reg),
10494 as_Register($src2$$reg),
10495 Assembler::LSR,
10496 $src3$$constant & 0x3f);
10497 %}
10498
10499 ins_pipe(ialu_reg_reg_shift);
10500 %}
10501
10502 // This pattern is automatically generated from aarch64_ad.m4
10503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10504 // val | (-1 ^ (val >> shift)) ==> ornw
10505 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10506 iRegIorL2I src1, iRegIorL2I src2,
10507 immI src3, immI_M1 src4) %{
10508 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10509 ins_cost(1.9 * INSN_COST);
10510 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10511
10512 ins_encode %{
10513 __ ornw(as_Register($dst$$reg),
10514 as_Register($src1$$reg),
10515 as_Register($src2$$reg),
10516 Assembler::ASR,
10517 $src3$$constant & 0x1f);
10518 %}
10519
10520 ins_pipe(ialu_reg_reg_shift);
10521 %}
10522
10523 // This pattern is automatically generated from aarch64_ad.m4
10524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10525 // val | (-1 ^ (val >> shift)) ==> orn
10526 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10527 iRegL src1, iRegL src2,
10528 immI src3, immL_M1 src4) %{
10529 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10530 ins_cost(1.9 * INSN_COST);
10531 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10532
10533 ins_encode %{
10534 __ orn(as_Register($dst$$reg),
10535 as_Register($src1$$reg),
10536 as_Register($src2$$reg),
10537 Assembler::ASR,
10538 $src3$$constant & 0x3f);
10539 %}
10540
10541 ins_pipe(ialu_reg_reg_shift);
10542 %}
10543
10544 // This pattern is automatically generated from aarch64_ad.m4
10545 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10546 // val | (-1 ^ (val ror shift)) ==> ornw
10547 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10548 iRegIorL2I src1, iRegIorL2I src2,
10549 immI src3, immI_M1 src4) %{
10550 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10551 ins_cost(1.9 * INSN_COST);
10552 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10553
10554 ins_encode %{
10555 __ ornw(as_Register($dst$$reg),
10556 as_Register($src1$$reg),
10557 as_Register($src2$$reg),
10558 Assembler::ROR,
10559 $src3$$constant & 0x1f);
10560 %}
10561
10562 ins_pipe(ialu_reg_reg_shift);
10563 %}
10564
10565 // This pattern is automatically generated from aarch64_ad.m4
10566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10567 // val | (-1 ^ (val ror shift)) ==> orn
10568 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10569 iRegL src1, iRegL src2,
10570 immI src3, immL_M1 src4) %{
10571 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10572 ins_cost(1.9 * INSN_COST);
10573 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10574
10575 ins_encode %{
10576 __ orn(as_Register($dst$$reg),
10577 as_Register($src1$$reg),
10578 as_Register($src2$$reg),
10579 Assembler::ROR,
10580 $src3$$constant & 0x3f);
10581 %}
10582
10583 ins_pipe(ialu_reg_reg_shift);
10584 %}
10585
10586 // This pattern is automatically generated from aarch64_ad.m4
10587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10588 // val | (-1 ^ (val << shift)) ==> ornw
10589 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10590 iRegIorL2I src1, iRegIorL2I src2,
10591 immI src3, immI_M1 src4) %{
10592 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10593 ins_cost(1.9 * INSN_COST);
10594 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10595
10596 ins_encode %{
10597 __ ornw(as_Register($dst$$reg),
10598 as_Register($src1$$reg),
10599 as_Register($src2$$reg),
10600 Assembler::LSL,
10601 $src3$$constant & 0x1f);
10602 %}
10603
10604 ins_pipe(ialu_reg_reg_shift);
10605 %}
10606
10607 // This pattern is automatically generated from aarch64_ad.m4
10608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10609 // val | (-1 ^ (val << shift)) ==> orn
10610 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10611 iRegL src1, iRegL src2,
10612 immI src3, immL_M1 src4) %{
10613 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10614 ins_cost(1.9 * INSN_COST);
10615 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10616
10617 ins_encode %{
10618 __ orn(as_Register($dst$$reg),
10619 as_Register($src1$$reg),
10620 as_Register($src2$$reg),
10621 Assembler::LSL,
10622 $src3$$constant & 0x3f);
10623 %}
10624
10625 ins_pipe(ialu_reg_reg_shift);
10626 %}
10627
10628 // This pattern is automatically generated from aarch64_ad.m4
10629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10630 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10631 iRegIorL2I src1, iRegIorL2I src2,
10632 immI src3) %{
10633 match(Set dst (AndI src1 (URShiftI src2 src3)));
10634
10635 ins_cost(1.9 * INSN_COST);
10636 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10637
10638 ins_encode %{
10639 __ andw(as_Register($dst$$reg),
10640 as_Register($src1$$reg),
10641 as_Register($src2$$reg),
10642 Assembler::LSR,
10643 $src3$$constant & 0x1f);
10644 %}
10645
10646 ins_pipe(ialu_reg_reg_shift);
10647 %}
10648
10649 // This pattern is automatically generated from aarch64_ad.m4
10650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10651 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10652 iRegL src1, iRegL src2,
10653 immI src3) %{
10654 match(Set dst (AndL src1 (URShiftL src2 src3)));
10655
10656 ins_cost(1.9 * INSN_COST);
10657 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10658
10659 ins_encode %{
10660 __ andr(as_Register($dst$$reg),
10661 as_Register($src1$$reg),
10662 as_Register($src2$$reg),
10663 Assembler::LSR,
10664 $src3$$constant & 0x3f);
10665 %}
10666
10667 ins_pipe(ialu_reg_reg_shift);
10668 %}
10669
10670 // This pattern is automatically generated from aarch64_ad.m4
10671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10672 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10673 iRegIorL2I src1, iRegIorL2I src2,
10674 immI src3) %{
10675 match(Set dst (AndI src1 (RShiftI src2 src3)));
10676
10677 ins_cost(1.9 * INSN_COST);
10678 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10679
10680 ins_encode %{
10681 __ andw(as_Register($dst$$reg),
10682 as_Register($src1$$reg),
10683 as_Register($src2$$reg),
10684 Assembler::ASR,
10685 $src3$$constant & 0x1f);
10686 %}
10687
10688 ins_pipe(ialu_reg_reg_shift);
10689 %}
10690
10691 // This pattern is automatically generated from aarch64_ad.m4
10692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10693 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10694 iRegL src1, iRegL src2,
10695 immI src3) %{
10696 match(Set dst (AndL src1 (RShiftL src2 src3)));
10697
10698 ins_cost(1.9 * INSN_COST);
10699 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10700
10701 ins_encode %{
10702 __ andr(as_Register($dst$$reg),
10703 as_Register($src1$$reg),
10704 as_Register($src2$$reg),
10705 Assembler::ASR,
10706 $src3$$constant & 0x3f);
10707 %}
10708
10709 ins_pipe(ialu_reg_reg_shift);
10710 %}
10711
10712 // This pattern is automatically generated from aarch64_ad.m4
10713 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10714 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10715 iRegIorL2I src1, iRegIorL2I src2,
10716 immI src3) %{
10717 match(Set dst (AndI src1 (LShiftI src2 src3)));
10718
10719 ins_cost(1.9 * INSN_COST);
10720 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10721
10722 ins_encode %{
10723 __ andw(as_Register($dst$$reg),
10724 as_Register($src1$$reg),
10725 as_Register($src2$$reg),
10726 Assembler::LSL,
10727 $src3$$constant & 0x1f);
10728 %}
10729
10730 ins_pipe(ialu_reg_reg_shift);
10731 %}
10732
10733 // This pattern is automatically generated from aarch64_ad.m4
10734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10735 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10736 iRegL src1, iRegL src2,
10737 immI src3) %{
10738 match(Set dst (AndL src1 (LShiftL src2 src3)));
10739
10740 ins_cost(1.9 * INSN_COST);
10741 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10742
10743 ins_encode %{
10744 __ andr(as_Register($dst$$reg),
10745 as_Register($src1$$reg),
10746 as_Register($src2$$reg),
10747 Assembler::LSL,
10748 $src3$$constant & 0x3f);
10749 %}
10750
10751 ins_pipe(ialu_reg_reg_shift);
10752 %}
10753
10754 // This pattern is automatically generated from aarch64_ad.m4
10755 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10756 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10757 iRegIorL2I src1, iRegIorL2I src2,
10758 immI src3) %{
10759 match(Set dst (AndI src1 (RotateRight src2 src3)));
10760
10761 ins_cost(1.9 * INSN_COST);
10762 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10763
10764 ins_encode %{
10765 __ andw(as_Register($dst$$reg),
10766 as_Register($src1$$reg),
10767 as_Register($src2$$reg),
10768 Assembler::ROR,
10769 $src3$$constant & 0x1f);
10770 %}
10771
10772 ins_pipe(ialu_reg_reg_shift);
10773 %}
10774
10775 // This pattern is automatically generated from aarch64_ad.m4
10776 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10777 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10778 iRegL src1, iRegL src2,
10779 immI src3) %{
10780 match(Set dst (AndL src1 (RotateRight src2 src3)));
10781
10782 ins_cost(1.9 * INSN_COST);
10783 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10784
10785 ins_encode %{
10786 __ andr(as_Register($dst$$reg),
10787 as_Register($src1$$reg),
10788 as_Register($src2$$reg),
10789 Assembler::ROR,
10790 $src3$$constant & 0x3f);
10791 %}
10792
10793 ins_pipe(ialu_reg_reg_shift);
10794 %}
10795
10796 // This pattern is automatically generated from aarch64_ad.m4
10797 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10798 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10799 iRegIorL2I src1, iRegIorL2I src2,
10800 immI src3) %{
10801 match(Set dst (XorI src1 (URShiftI src2 src3)));
10802
10803 ins_cost(1.9 * INSN_COST);
10804 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10805
10806 ins_encode %{
10807 __ eorw(as_Register($dst$$reg),
10808 as_Register($src1$$reg),
10809 as_Register($src2$$reg),
10810 Assembler::LSR,
10811 $src3$$constant & 0x1f);
10812 %}
10813
10814 ins_pipe(ialu_reg_reg_shift);
10815 %}
10816
10817 // This pattern is automatically generated from aarch64_ad.m4
10818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10819 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10820 iRegL src1, iRegL src2,
10821 immI src3) %{
10822 match(Set dst (XorL src1 (URShiftL src2 src3)));
10823
10824 ins_cost(1.9 * INSN_COST);
10825 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10826
10827 ins_encode %{
10828 __ eor(as_Register($dst$$reg),
10829 as_Register($src1$$reg),
10830 as_Register($src2$$reg),
10831 Assembler::LSR,
10832 $src3$$constant & 0x3f);
10833 %}
10834
10835 ins_pipe(ialu_reg_reg_shift);
10836 %}
10837
10838 // This pattern is automatically generated from aarch64_ad.m4
10839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10840 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10841 iRegIorL2I src1, iRegIorL2I src2,
10842 immI src3) %{
10843 match(Set dst (XorI src1 (RShiftI src2 src3)));
10844
10845 ins_cost(1.9 * INSN_COST);
10846 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10847
10848 ins_encode %{
10849 __ eorw(as_Register($dst$$reg),
10850 as_Register($src1$$reg),
10851 as_Register($src2$$reg),
10852 Assembler::ASR,
10853 $src3$$constant & 0x1f);
10854 %}
10855
10856 ins_pipe(ialu_reg_reg_shift);
10857 %}
10858
10859 // This pattern is automatically generated from aarch64_ad.m4
10860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10861 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10862 iRegL src1, iRegL src2,
10863 immI src3) %{
10864 match(Set dst (XorL src1 (RShiftL src2 src3)));
10865
10866 ins_cost(1.9 * INSN_COST);
10867 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10868
10869 ins_encode %{
10870 __ eor(as_Register($dst$$reg),
10871 as_Register($src1$$reg),
10872 as_Register($src2$$reg),
10873 Assembler::ASR,
10874 $src3$$constant & 0x3f);
10875 %}
10876
10877 ins_pipe(ialu_reg_reg_shift);
10878 %}
10879
10880 // This pattern is automatically generated from aarch64_ad.m4
10881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10882 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10883 iRegIorL2I src1, iRegIorL2I src2,
10884 immI src3) %{
10885 match(Set dst (XorI src1 (LShiftI src2 src3)));
10886
10887 ins_cost(1.9 * INSN_COST);
10888 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10889
10890 ins_encode %{
10891 __ eorw(as_Register($dst$$reg),
10892 as_Register($src1$$reg),
10893 as_Register($src2$$reg),
10894 Assembler::LSL,
10895 $src3$$constant & 0x1f);
10896 %}
10897
10898 ins_pipe(ialu_reg_reg_shift);
10899 %}
10900
10901 // This pattern is automatically generated from aarch64_ad.m4
10902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10903 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10904 iRegL src1, iRegL src2,
10905 immI src3) %{
10906 match(Set dst (XorL src1 (LShiftL src2 src3)));
10907
10908 ins_cost(1.9 * INSN_COST);
10909 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10910
10911 ins_encode %{
10912 __ eor(as_Register($dst$$reg),
10913 as_Register($src1$$reg),
10914 as_Register($src2$$reg),
10915 Assembler::LSL,
10916 $src3$$constant & 0x3f);
10917 %}
10918
10919 ins_pipe(ialu_reg_reg_shift);
10920 %}
10921
10922 // This pattern is automatically generated from aarch64_ad.m4
10923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10924 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10925 iRegIorL2I src1, iRegIorL2I src2,
10926 immI src3) %{
10927 match(Set dst (XorI src1 (RotateRight src2 src3)));
10928
10929 ins_cost(1.9 * INSN_COST);
10930 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10931
10932 ins_encode %{
10933 __ eorw(as_Register($dst$$reg),
10934 as_Register($src1$$reg),
10935 as_Register($src2$$reg),
10936 Assembler::ROR,
10937 $src3$$constant & 0x1f);
10938 %}
10939
10940 ins_pipe(ialu_reg_reg_shift);
10941 %}
10942
10943 // This pattern is automatically generated from aarch64_ad.m4
10944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10945 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10946 iRegL src1, iRegL src2,
10947 immI src3) %{
10948 match(Set dst (XorL src1 (RotateRight src2 src3)));
10949
10950 ins_cost(1.9 * INSN_COST);
10951 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10952
10953 ins_encode %{
10954 __ eor(as_Register($dst$$reg),
10955 as_Register($src1$$reg),
10956 as_Register($src2$$reg),
10957 Assembler::ROR,
10958 $src3$$constant & 0x3f);
10959 %}
10960
10961 ins_pipe(ialu_reg_reg_shift);
10962 %}
10963
10964 // This pattern is automatically generated from aarch64_ad.m4
10965 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10966 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10967 iRegIorL2I src1, iRegIorL2I src2,
10968 immI src3) %{
10969 match(Set dst (OrI src1 (URShiftI src2 src3)));
10970
10971 ins_cost(1.9 * INSN_COST);
10972 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10973
10974 ins_encode %{
10975 __ orrw(as_Register($dst$$reg),
10976 as_Register($src1$$reg),
10977 as_Register($src2$$reg),
10978 Assembler::LSR,
10979 $src3$$constant & 0x1f);
10980 %}
10981
10982 ins_pipe(ialu_reg_reg_shift);
10983 %}
10984
10985 // This pattern is automatically generated from aarch64_ad.m4
10986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10987 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10988 iRegL src1, iRegL src2,
10989 immI src3) %{
10990 match(Set dst (OrL src1 (URShiftL src2 src3)));
10991
10992 ins_cost(1.9 * INSN_COST);
10993 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10994
10995 ins_encode %{
10996 __ orr(as_Register($dst$$reg),
10997 as_Register($src1$$reg),
10998 as_Register($src2$$reg),
10999 Assembler::LSR,
11000 $src3$$constant & 0x3f);
11001 %}
11002
11003 ins_pipe(ialu_reg_reg_shift);
11004 %}
11005
11006 // This pattern is automatically generated from aarch64_ad.m4
11007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11008 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11009 iRegIorL2I src1, iRegIorL2I src2,
11010 immI src3) %{
11011 match(Set dst (OrI src1 (RShiftI src2 src3)));
11012
11013 ins_cost(1.9 * INSN_COST);
11014 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11015
11016 ins_encode %{
11017 __ orrw(as_Register($dst$$reg),
11018 as_Register($src1$$reg),
11019 as_Register($src2$$reg),
11020 Assembler::ASR,
11021 $src3$$constant & 0x1f);
11022 %}
11023
11024 ins_pipe(ialu_reg_reg_shift);
11025 %}
11026
11027 // This pattern is automatically generated from aarch64_ad.m4
11028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11029 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11030 iRegL src1, iRegL src2,
11031 immI src3) %{
11032 match(Set dst (OrL src1 (RShiftL src2 src3)));
11033
11034 ins_cost(1.9 * INSN_COST);
11035 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11036
11037 ins_encode %{
11038 __ orr(as_Register($dst$$reg),
11039 as_Register($src1$$reg),
11040 as_Register($src2$$reg),
11041 Assembler::ASR,
11042 $src3$$constant & 0x3f);
11043 %}
11044
11045 ins_pipe(ialu_reg_reg_shift);
11046 %}
11047
11048 // This pattern is automatically generated from aarch64_ad.m4
11049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11050 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11051 iRegIorL2I src1, iRegIorL2I src2,
11052 immI src3) %{
11053 match(Set dst (OrI src1 (LShiftI src2 src3)));
11054
11055 ins_cost(1.9 * INSN_COST);
11056 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11057
11058 ins_encode %{
11059 __ orrw(as_Register($dst$$reg),
11060 as_Register($src1$$reg),
11061 as_Register($src2$$reg),
11062 Assembler::LSL,
11063 $src3$$constant & 0x1f);
11064 %}
11065
11066 ins_pipe(ialu_reg_reg_shift);
11067 %}
11068
11069 // This pattern is automatically generated from aarch64_ad.m4
11070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11071 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11072 iRegL src1, iRegL src2,
11073 immI src3) %{
11074 match(Set dst (OrL src1 (LShiftL src2 src3)));
11075
11076 ins_cost(1.9 * INSN_COST);
11077 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11078
11079 ins_encode %{
11080 __ orr(as_Register($dst$$reg),
11081 as_Register($src1$$reg),
11082 as_Register($src2$$reg),
11083 Assembler::LSL,
11084 $src3$$constant & 0x3f);
11085 %}
11086
11087 ins_pipe(ialu_reg_reg_shift);
11088 %}
11089
11090 // This pattern is automatically generated from aarch64_ad.m4
11091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11092 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11093 iRegIorL2I src1, iRegIorL2I src2,
11094 immI src3) %{
11095 match(Set dst (OrI src1 (RotateRight src2 src3)));
11096
11097 ins_cost(1.9 * INSN_COST);
11098 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11099
11100 ins_encode %{
11101 __ orrw(as_Register($dst$$reg),
11102 as_Register($src1$$reg),
11103 as_Register($src2$$reg),
11104 Assembler::ROR,
11105 $src3$$constant & 0x1f);
11106 %}
11107
11108 ins_pipe(ialu_reg_reg_shift);
11109 %}
11110
11111 // This pattern is automatically generated from aarch64_ad.m4
11112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11113 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11114 iRegL src1, iRegL src2,
11115 immI src3) %{
11116 match(Set dst (OrL src1 (RotateRight src2 src3)));
11117
11118 ins_cost(1.9 * INSN_COST);
11119 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11120
11121 ins_encode %{
11122 __ orr(as_Register($dst$$reg),
11123 as_Register($src1$$reg),
11124 as_Register($src2$$reg),
11125 Assembler::ROR,
11126 $src3$$constant & 0x3f);
11127 %}
11128
11129 ins_pipe(ialu_reg_reg_shift);
11130 %}
11131
11132 // This pattern is automatically generated from aarch64_ad.m4
11133 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11134 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11135 iRegIorL2I src1, iRegIorL2I src2,
11136 immI src3) %{
11137 match(Set dst (AddI src1 (URShiftI src2 src3)));
11138
11139 ins_cost(1.9 * INSN_COST);
11140 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11141
11142 ins_encode %{
11143 __ addw(as_Register($dst$$reg),
11144 as_Register($src1$$reg),
11145 as_Register($src2$$reg),
11146 Assembler::LSR,
11147 $src3$$constant & 0x1f);
11148 %}
11149
11150 ins_pipe(ialu_reg_reg_shift);
11151 %}
11152
11153 // This pattern is automatically generated from aarch64_ad.m4
11154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11155 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11156 iRegL src1, iRegL src2,
11157 immI src3) %{
11158 match(Set dst (AddL src1 (URShiftL src2 src3)));
11159
11160 ins_cost(1.9 * INSN_COST);
11161 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11162
11163 ins_encode %{
11164 __ add(as_Register($dst$$reg),
11165 as_Register($src1$$reg),
11166 as_Register($src2$$reg),
11167 Assembler::LSR,
11168 $src3$$constant & 0x3f);
11169 %}
11170
11171 ins_pipe(ialu_reg_reg_shift);
11172 %}
11173
11174 // This pattern is automatically generated from aarch64_ad.m4
11175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11176 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11177 iRegIorL2I src1, iRegIorL2I src2,
11178 immI src3) %{
11179 match(Set dst (AddI src1 (RShiftI src2 src3)));
11180
11181 ins_cost(1.9 * INSN_COST);
11182 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11183
11184 ins_encode %{
11185 __ addw(as_Register($dst$$reg),
11186 as_Register($src1$$reg),
11187 as_Register($src2$$reg),
11188 Assembler::ASR,
11189 $src3$$constant & 0x1f);
11190 %}
11191
11192 ins_pipe(ialu_reg_reg_shift);
11193 %}
11194
11195 // This pattern is automatically generated from aarch64_ad.m4
11196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11197 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11198 iRegL src1, iRegL src2,
11199 immI src3) %{
11200 match(Set dst (AddL src1 (RShiftL src2 src3)));
11201
11202 ins_cost(1.9 * INSN_COST);
11203 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11204
11205 ins_encode %{
11206 __ add(as_Register($dst$$reg),
11207 as_Register($src1$$reg),
11208 as_Register($src2$$reg),
11209 Assembler::ASR,
11210 $src3$$constant & 0x3f);
11211 %}
11212
11213 ins_pipe(ialu_reg_reg_shift);
11214 %}
11215
11216 // This pattern is automatically generated from aarch64_ad.m4
11217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11218 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11219 iRegIorL2I src1, iRegIorL2I src2,
11220 immI src3) %{
11221 match(Set dst (AddI src1 (LShiftI src2 src3)));
11222
11223 ins_cost(1.9 * INSN_COST);
11224 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11225
11226 ins_encode %{
11227 __ addw(as_Register($dst$$reg),
11228 as_Register($src1$$reg),
11229 as_Register($src2$$reg),
11230 Assembler::LSL,
11231 $src3$$constant & 0x1f);
11232 %}
11233
11234 ins_pipe(ialu_reg_reg_shift);
11235 %}
11236
11237 // This pattern is automatically generated from aarch64_ad.m4
11238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11239 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11240 iRegL src1, iRegL src2,
11241 immI src3) %{
11242 match(Set dst (AddL src1 (LShiftL src2 src3)));
11243
11244 ins_cost(1.9 * INSN_COST);
11245 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11246
11247 ins_encode %{
11248 __ add(as_Register($dst$$reg),
11249 as_Register($src1$$reg),
11250 as_Register($src2$$reg),
11251 Assembler::LSL,
11252 $src3$$constant & 0x3f);
11253 %}
11254
11255 ins_pipe(ialu_reg_reg_shift);
11256 %}
11257
11258 // This pattern is automatically generated from aarch64_ad.m4
11259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11260 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11261 iRegIorL2I src1, iRegIorL2I src2,
11262 immI src3) %{
11263 match(Set dst (SubI src1 (URShiftI src2 src3)));
11264
11265 ins_cost(1.9 * INSN_COST);
11266 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11267
11268 ins_encode %{
11269 __ subw(as_Register($dst$$reg),
11270 as_Register($src1$$reg),
11271 as_Register($src2$$reg),
11272 Assembler::LSR,
11273 $src3$$constant & 0x1f);
11274 %}
11275
11276 ins_pipe(ialu_reg_reg_shift);
11277 %}
11278
11279 // This pattern is automatically generated from aarch64_ad.m4
11280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11281 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11282 iRegL src1, iRegL src2,
11283 immI src3) %{
11284 match(Set dst (SubL src1 (URShiftL src2 src3)));
11285
11286 ins_cost(1.9 * INSN_COST);
11287 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11288
11289 ins_encode %{
11290 __ sub(as_Register($dst$$reg),
11291 as_Register($src1$$reg),
11292 as_Register($src2$$reg),
11293 Assembler::LSR,
11294 $src3$$constant & 0x3f);
11295 %}
11296
11297 ins_pipe(ialu_reg_reg_shift);
11298 %}
11299
11300 // This pattern is automatically generated from aarch64_ad.m4
11301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11302 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11303 iRegIorL2I src1, iRegIorL2I src2,
11304 immI src3) %{
11305 match(Set dst (SubI src1 (RShiftI src2 src3)));
11306
11307 ins_cost(1.9 * INSN_COST);
11308 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11309
11310 ins_encode %{
11311 __ subw(as_Register($dst$$reg),
11312 as_Register($src1$$reg),
11313 as_Register($src2$$reg),
11314 Assembler::ASR,
11315 $src3$$constant & 0x1f);
11316 %}
11317
11318 ins_pipe(ialu_reg_reg_shift);
11319 %}
11320
11321 // This pattern is automatically generated from aarch64_ad.m4
11322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11323 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11324 iRegL src1, iRegL src2,
11325 immI src3) %{
11326 match(Set dst (SubL src1 (RShiftL src2 src3)));
11327
11328 ins_cost(1.9 * INSN_COST);
11329 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11330
11331 ins_encode %{
11332 __ sub(as_Register($dst$$reg),
11333 as_Register($src1$$reg),
11334 as_Register($src2$$reg),
11335 Assembler::ASR,
11336 $src3$$constant & 0x3f);
11337 %}
11338
11339 ins_pipe(ialu_reg_reg_shift);
11340 %}
11341
11342 // This pattern is automatically generated from aarch64_ad.m4
11343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11344 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11345 iRegIorL2I src1, iRegIorL2I src2,
11346 immI src3) %{
11347 match(Set dst (SubI src1 (LShiftI src2 src3)));
11348
11349 ins_cost(1.9 * INSN_COST);
11350 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11351
11352 ins_encode %{
11353 __ subw(as_Register($dst$$reg),
11354 as_Register($src1$$reg),
11355 as_Register($src2$$reg),
11356 Assembler::LSL,
11357 $src3$$constant & 0x1f);
11358 %}
11359
11360 ins_pipe(ialu_reg_reg_shift);
11361 %}
11362
11363 // This pattern is automatically generated from aarch64_ad.m4
11364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11365 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11366 iRegL src1, iRegL src2,
11367 immI src3) %{
11368 match(Set dst (SubL src1 (LShiftL src2 src3)));
11369
11370 ins_cost(1.9 * INSN_COST);
11371 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11372
11373 ins_encode %{
11374 __ sub(as_Register($dst$$reg),
11375 as_Register($src1$$reg),
11376 as_Register($src2$$reg),
11377 Assembler::LSL,
11378 $src3$$constant & 0x3f);
11379 %}
11380
11381 ins_pipe(ialu_reg_reg_shift);
11382 %}
11383
11384 // This pattern is automatically generated from aarch64_ad.m4
11385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11386
11387 // Shift Left followed by Shift Right.
11388 // This idiom is used by the compiler for the i2b bytecode etc.
11389 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11390 %{
11391 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11392 ins_cost(INSN_COST * 2);
11393 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11394 ins_encode %{
11395 int lshift = $lshift_count$$constant & 63;
11396 int rshift = $rshift_count$$constant & 63;
11397 int s = 63 - lshift;
11398 int r = (rshift - lshift) & 63;
11399 __ sbfm(as_Register($dst$$reg),
11400 as_Register($src$$reg),
11401 r, s);
11402 %}
11403
11404 ins_pipe(ialu_reg_shift);
11405 %}
11406
11407 // This pattern is automatically generated from aarch64_ad.m4
11408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11409
11410 // Shift Left followed by Shift Right.
11411 // This idiom is used by the compiler for the i2b bytecode etc.
11412 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11413 %{
11414 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11415 ins_cost(INSN_COST * 2);
11416 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11417 ins_encode %{
11418 int lshift = $lshift_count$$constant & 31;
11419 int rshift = $rshift_count$$constant & 31;
11420 int s = 31 - lshift;
11421 int r = (rshift - lshift) & 31;
11422 __ sbfmw(as_Register($dst$$reg),
11423 as_Register($src$$reg),
11424 r, s);
11425 %}
11426
11427 ins_pipe(ialu_reg_shift);
11428 %}
11429
11430 // This pattern is automatically generated from aarch64_ad.m4
11431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11432
11433 // Shift Left followed by Shift Right.
11434 // This idiom is used by the compiler for the i2b bytecode etc.
11435 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11436 %{
11437 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11438 ins_cost(INSN_COST * 2);
11439 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11440 ins_encode %{
11441 int lshift = $lshift_count$$constant & 63;
11442 int rshift = $rshift_count$$constant & 63;
11443 int s = 63 - lshift;
11444 int r = (rshift - lshift) & 63;
11445 __ ubfm(as_Register($dst$$reg),
11446 as_Register($src$$reg),
11447 r, s);
11448 %}
11449
11450 ins_pipe(ialu_reg_shift);
11451 %}
11452
11453 // This pattern is automatically generated from aarch64_ad.m4
11454 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11455
11456 // Shift Left followed by Shift Right.
11457 // This idiom is used by the compiler for the i2b bytecode etc.
11458 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11459 %{
11460 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11461 ins_cost(INSN_COST * 2);
11462 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11463 ins_encode %{
11464 int lshift = $lshift_count$$constant & 31;
11465 int rshift = $rshift_count$$constant & 31;
11466 int s = 31 - lshift;
11467 int r = (rshift - lshift) & 31;
11468 __ ubfmw(as_Register($dst$$reg),
11469 as_Register($src$$reg),
11470 r, s);
11471 %}
11472
11473 ins_pipe(ialu_reg_shift);
11474 %}
11475
11476 // Bitfield extract with shift & mask
11477
11478 // This pattern is automatically generated from aarch64_ad.m4
11479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11480 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11481 %{
11482 match(Set dst (AndI (URShiftI src rshift) mask));
11483 // Make sure we are not going to exceed what ubfxw can do.
11484 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11485
11486 ins_cost(INSN_COST);
11487 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11488 ins_encode %{
11489 int rshift = $rshift$$constant & 31;
11490 intptr_t mask = $mask$$constant;
11491 int width = exact_log2(mask+1);
11492 __ ubfxw(as_Register($dst$$reg),
11493 as_Register($src$$reg), rshift, width);
11494 %}
11495 ins_pipe(ialu_reg_shift);
11496 %}
11497
11498 // This pattern is automatically generated from aarch64_ad.m4
11499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11500 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11501 %{
11502 match(Set dst (AndL (URShiftL src rshift) mask));
11503 // Make sure we are not going to exceed what ubfx can do.
11504 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11505
11506 ins_cost(INSN_COST);
11507 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11508 ins_encode %{
11509 int rshift = $rshift$$constant & 63;
11510 intptr_t mask = $mask$$constant;
11511 int width = exact_log2_long(mask+1);
11512 __ ubfx(as_Register($dst$$reg),
11513 as_Register($src$$reg), rshift, width);
11514 %}
11515 ins_pipe(ialu_reg_shift);
11516 %}
11517
11518
11519 // This pattern is automatically generated from aarch64_ad.m4
11520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11521
11522 // We can use ubfx when extending an And with a mask when we know mask
11523 // is positive. We know that because immI_bitmask guarantees it.
11524 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11525 %{
11526 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11527 // Make sure we are not going to exceed what ubfxw can do.
11528 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11529
11530 ins_cost(INSN_COST * 2);
11531 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11532 ins_encode %{
11533 int rshift = $rshift$$constant & 31;
11534 intptr_t mask = $mask$$constant;
11535 int width = exact_log2(mask+1);
11536 __ ubfx(as_Register($dst$$reg),
11537 as_Register($src$$reg), rshift, width);
11538 %}
11539 ins_pipe(ialu_reg_shift);
11540 %}
11541
11542
11543 // This pattern is automatically generated from aarch64_ad.m4
11544 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11545
11546 // We can use ubfiz when masking by a positive number and then left shifting the result.
11547 // We know that the mask is positive because immI_bitmask guarantees it.
11548 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11549 %{
11550 match(Set dst (LShiftI (AndI src mask) lshift));
11551 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11552
11553 ins_cost(INSN_COST);
11554 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11555 ins_encode %{
11556 int lshift = $lshift$$constant & 31;
11557 intptr_t mask = $mask$$constant;
11558 int width = exact_log2(mask+1);
11559 __ ubfizw(as_Register($dst$$reg),
11560 as_Register($src$$reg), lshift, width);
11561 %}
11562 ins_pipe(ialu_reg_shift);
11563 %}
11564
11565 // This pattern is automatically generated from aarch64_ad.m4
11566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11567
11568 // We can use ubfiz when masking by a positive number and then left shifting the result.
11569 // We know that the mask is positive because immL_bitmask guarantees it.
11570 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11571 %{
11572 match(Set dst (LShiftL (AndL src mask) lshift));
11573 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11574
11575 ins_cost(INSN_COST);
11576 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11577 ins_encode %{
11578 int lshift = $lshift$$constant & 63;
11579 intptr_t mask = $mask$$constant;
11580 int width = exact_log2_long(mask+1);
11581 __ ubfiz(as_Register($dst$$reg),
11582 as_Register($src$$reg), lshift, width);
11583 %}
11584 ins_pipe(ialu_reg_shift);
11585 %}
11586
11587 // This pattern is automatically generated from aarch64_ad.m4
11588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11589
11590 // We can use ubfiz when masking by a positive number and then left shifting the result.
11591 // We know that the mask is positive because immI_bitmask guarantees it.
11592 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11593 %{
11594 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11595 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11596
11597 ins_cost(INSN_COST);
11598 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11599 ins_encode %{
11600 int lshift = $lshift$$constant & 31;
11601 intptr_t mask = $mask$$constant;
11602 int width = exact_log2(mask+1);
11603 __ ubfizw(as_Register($dst$$reg),
11604 as_Register($src$$reg), lshift, width);
11605 %}
11606 ins_pipe(ialu_reg_shift);
11607 %}
11608
11609 // This pattern is automatically generated from aarch64_ad.m4
11610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11611
11612 // We can use ubfiz when masking by a positive number and then left shifting the result.
11613 // We know that the mask is positive because immL_bitmask guarantees it.
11614 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11615 %{
11616 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11617 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11618
11619 ins_cost(INSN_COST);
11620 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11621 ins_encode %{
11622 int lshift = $lshift$$constant & 63;
11623 intptr_t mask = $mask$$constant;
11624 int width = exact_log2_long(mask+1);
11625 __ ubfiz(as_Register($dst$$reg),
11626 as_Register($src$$reg), lshift, width);
11627 %}
11628 ins_pipe(ialu_reg_shift);
11629 %}
11630
11631
11632 // This pattern is automatically generated from aarch64_ad.m4
11633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11634
11635 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11636 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11637 %{
11638 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11639 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11640
11641 ins_cost(INSN_COST);
11642 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11643 ins_encode %{
11644 int lshift = $lshift$$constant & 63;
11645 intptr_t mask = $mask$$constant;
11646 int width = exact_log2(mask+1);
11647 __ ubfiz(as_Register($dst$$reg),
11648 as_Register($src$$reg), lshift, width);
11649 %}
11650 ins_pipe(ialu_reg_shift);
11651 %}
11652
11653 // This pattern is automatically generated from aarch64_ad.m4
11654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11655
11656 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11657 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11658 %{
11659 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11660 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11661
11662 ins_cost(INSN_COST);
11663 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11664 ins_encode %{
11665 int lshift = $lshift$$constant & 31;
11666 intptr_t mask = $mask$$constant;
11667 int width = exact_log2(mask+1);
11668 __ ubfiz(as_Register($dst$$reg),
11669 as_Register($src$$reg), lshift, width);
11670 %}
11671 ins_pipe(ialu_reg_shift);
11672 %}
11673
11674 // This pattern is automatically generated from aarch64_ad.m4
11675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11676
11677 // Can skip int2long conversions after AND with small bitmask
11678 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11679 %{
11680 match(Set dst (ConvI2L (AndI src msk)));
11681 ins_cost(INSN_COST);
11682 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11683 ins_encode %{
11684 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11685 %}
11686 ins_pipe(ialu_reg_shift);
11687 %}
11688
11689
11690 // Rotations
11691
11692 // This pattern is automatically generated from aarch64_ad.m4
11693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11694 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11695 %{
11696 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11697 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11698
11699 ins_cost(INSN_COST);
11700 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11701
11702 ins_encode %{
11703 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11704 $rshift$$constant & 63);
11705 %}
11706 ins_pipe(ialu_reg_reg_extr);
11707 %}
11708
11709
11710 // This pattern is automatically generated from aarch64_ad.m4
11711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11712 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11713 %{
11714 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11715 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11716
11717 ins_cost(INSN_COST);
11718 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11719
11720 ins_encode %{
11721 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11722 $rshift$$constant & 31);
11723 %}
11724 ins_pipe(ialu_reg_reg_extr);
11725 %}
11726
11727
11728 // This pattern is automatically generated from aarch64_ad.m4
11729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11730 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11731 %{
11732 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11733 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11734
11735 ins_cost(INSN_COST);
11736 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11737
11738 ins_encode %{
11739 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11740 $rshift$$constant & 63);
11741 %}
11742 ins_pipe(ialu_reg_reg_extr);
11743 %}
11744
11745
11746 // This pattern is automatically generated from aarch64_ad.m4
11747 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11748 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11749 %{
11750 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11751 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11752
11753 ins_cost(INSN_COST);
11754 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11755
11756 ins_encode %{
11757 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11758 $rshift$$constant & 31);
11759 %}
11760 ins_pipe(ialu_reg_reg_extr);
11761 %}
11762
11763 // This pattern is automatically generated from aarch64_ad.m4
11764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11765 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11766 %{
11767 match(Set dst (RotateRight src shift));
11768
11769 ins_cost(INSN_COST);
11770 format %{ "ror $dst, $src, $shift" %}
11771
11772 ins_encode %{
11773 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11774 $shift$$constant & 0x1f);
11775 %}
11776 ins_pipe(ialu_reg_reg_vshift);
11777 %}
11778
11779 // This pattern is automatically generated from aarch64_ad.m4
11780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11781 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11782 %{
11783 match(Set dst (RotateRight src shift));
11784
11785 ins_cost(INSN_COST);
11786 format %{ "ror $dst, $src, $shift" %}
11787
11788 ins_encode %{
11789 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11790 $shift$$constant & 0x3f);
11791 %}
11792 ins_pipe(ialu_reg_reg_vshift);
11793 %}
11794
11795 // This pattern is automatically generated from aarch64_ad.m4
11796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11797 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11798 %{
11799 match(Set dst (RotateRight src shift));
11800
11801 ins_cost(INSN_COST);
11802 format %{ "ror $dst, $src, $shift" %}
11803
11804 ins_encode %{
11805 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11806 %}
11807 ins_pipe(ialu_reg_reg_vshift);
11808 %}
11809
11810 // This pattern is automatically generated from aarch64_ad.m4
11811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11812 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11813 %{
11814 match(Set dst (RotateRight src shift));
11815
11816 ins_cost(INSN_COST);
11817 format %{ "ror $dst, $src, $shift" %}
11818
11819 ins_encode %{
11820 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11821 %}
11822 ins_pipe(ialu_reg_reg_vshift);
11823 %}
11824
11825 // This pattern is automatically generated from aarch64_ad.m4
11826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11827 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11828 %{
11829 match(Set dst (RotateLeft src shift));
11830
11831 ins_cost(INSN_COST);
11832 format %{ "rol $dst, $src, $shift" %}
11833
11834 ins_encode %{
11835 __ subw(rscratch1, zr, as_Register($shift$$reg));
11836 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11837 %}
11838 ins_pipe(ialu_reg_reg_vshift);
11839 %}
11840
11841 // This pattern is automatically generated from aarch64_ad.m4
11842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11843 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11844 %{
11845 match(Set dst (RotateLeft src shift));
11846
11847 ins_cost(INSN_COST);
11848 format %{ "rol $dst, $src, $shift" %}
11849
11850 ins_encode %{
11851 __ subw(rscratch1, zr, as_Register($shift$$reg));
11852 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11853 %}
11854 ins_pipe(ialu_reg_reg_vshift);
11855 %}
11856
11857
11858 // Add/subtract (extended)
11859
11860 // This pattern is automatically generated from aarch64_ad.m4
11861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11862 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11863 %{
11864 match(Set dst (AddL src1 (ConvI2L src2)));
11865 ins_cost(INSN_COST);
11866 format %{ "add $dst, $src1, $src2, sxtw" %}
11867
11868 ins_encode %{
11869 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11870 as_Register($src2$$reg), ext::sxtw);
11871 %}
11872 ins_pipe(ialu_reg_reg);
11873 %}
11874
11875 // This pattern is automatically generated from aarch64_ad.m4
11876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11877 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11878 %{
11879 match(Set dst (SubL src1 (ConvI2L src2)));
11880 ins_cost(INSN_COST);
11881 format %{ "sub $dst, $src1, $src2, sxtw" %}
11882
11883 ins_encode %{
11884 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11885 as_Register($src2$$reg), ext::sxtw);
11886 %}
11887 ins_pipe(ialu_reg_reg);
11888 %}
11889
11890 // This pattern is automatically generated from aarch64_ad.m4
11891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11892 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11893 %{
11894 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11895 ins_cost(INSN_COST);
11896 format %{ "add $dst, $src1, $src2, sxth" %}
11897
11898 ins_encode %{
11899 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11900 as_Register($src2$$reg), ext::sxth);
11901 %}
11902 ins_pipe(ialu_reg_reg);
11903 %}
11904
11905 // This pattern is automatically generated from aarch64_ad.m4
11906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11907 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11908 %{
11909 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11910 ins_cost(INSN_COST);
11911 format %{ "add $dst, $src1, $src2, sxtb" %}
11912
11913 ins_encode %{
11914 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11915 as_Register($src2$$reg), ext::sxtb);
11916 %}
11917 ins_pipe(ialu_reg_reg);
11918 %}
11919
11920 // This pattern is automatically generated from aarch64_ad.m4
11921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11922 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11923 %{
11924 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11925 ins_cost(INSN_COST);
11926 format %{ "add $dst, $src1, $src2, uxtb" %}
11927
11928 ins_encode %{
11929 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11930 as_Register($src2$$reg), ext::uxtb);
11931 %}
11932 ins_pipe(ialu_reg_reg);
11933 %}
11934
11935 // This pattern is automatically generated from aarch64_ad.m4
11936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11937 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11938 %{
11939 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11940 ins_cost(INSN_COST);
11941 format %{ "add $dst, $src1, $src2, sxth" %}
11942
11943 ins_encode %{
11944 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11945 as_Register($src2$$reg), ext::sxth);
11946 %}
11947 ins_pipe(ialu_reg_reg);
11948 %}
11949
11950 // This pattern is automatically generated from aarch64_ad.m4
11951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11952 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11953 %{
11954 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11955 ins_cost(INSN_COST);
11956 format %{ "add $dst, $src1, $src2, sxtw" %}
11957
11958 ins_encode %{
11959 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11960 as_Register($src2$$reg), ext::sxtw);
11961 %}
11962 ins_pipe(ialu_reg_reg);
11963 %}
11964
11965 // This pattern is automatically generated from aarch64_ad.m4
11966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11967 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11968 %{
11969 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11970 ins_cost(INSN_COST);
11971 format %{ "add $dst, $src1, $src2, sxtb" %}
11972
11973 ins_encode %{
11974 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11975 as_Register($src2$$reg), ext::sxtb);
11976 %}
11977 ins_pipe(ialu_reg_reg);
11978 %}
11979
11980 // This pattern is automatically generated from aarch64_ad.m4
11981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11982 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11983 %{
11984 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11985 ins_cost(INSN_COST);
11986 format %{ "add $dst, $src1, $src2, uxtb" %}
11987
11988 ins_encode %{
11989 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11990 as_Register($src2$$reg), ext::uxtb);
11991 %}
11992 ins_pipe(ialu_reg_reg);
11993 %}
11994
11995 // This pattern is automatically generated from aarch64_ad.m4
11996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11997 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11998 %{
11999 match(Set dst (AddI src1 (AndI src2 mask)));
12000 ins_cost(INSN_COST);
12001 format %{ "addw $dst, $src1, $src2, uxtb" %}
12002
12003 ins_encode %{
12004 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12005 as_Register($src2$$reg), ext::uxtb);
12006 %}
12007 ins_pipe(ialu_reg_reg);
12008 %}
12009
12010 // This pattern is automatically generated from aarch64_ad.m4
12011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12012 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12013 %{
12014 match(Set dst (AddI src1 (AndI src2 mask)));
12015 ins_cost(INSN_COST);
12016 format %{ "addw $dst, $src1, $src2, uxth" %}
12017
12018 ins_encode %{
12019 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12020 as_Register($src2$$reg), ext::uxth);
12021 %}
12022 ins_pipe(ialu_reg_reg);
12023 %}
12024
12025 // This pattern is automatically generated from aarch64_ad.m4
12026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12027 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12028 %{
12029 match(Set dst (AddL src1 (AndL src2 mask)));
12030 ins_cost(INSN_COST);
12031 format %{ "add $dst, $src1, $src2, uxtb" %}
12032
12033 ins_encode %{
12034 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12035 as_Register($src2$$reg), ext::uxtb);
12036 %}
12037 ins_pipe(ialu_reg_reg);
12038 %}
12039
12040 // This pattern is automatically generated from aarch64_ad.m4
12041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12042 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12043 %{
12044 match(Set dst (AddL src1 (AndL src2 mask)));
12045 ins_cost(INSN_COST);
12046 format %{ "add $dst, $src1, $src2, uxth" %}
12047
12048 ins_encode %{
12049 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12050 as_Register($src2$$reg), ext::uxth);
12051 %}
12052 ins_pipe(ialu_reg_reg);
12053 %}
12054
12055 // This pattern is automatically generated from aarch64_ad.m4
12056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12057 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12058 %{
12059 match(Set dst (AddL src1 (AndL src2 mask)));
12060 ins_cost(INSN_COST);
12061 format %{ "add $dst, $src1, $src2, uxtw" %}
12062
12063 ins_encode %{
12064 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12065 as_Register($src2$$reg), ext::uxtw);
12066 %}
12067 ins_pipe(ialu_reg_reg);
12068 %}
12069
12070 // This pattern is automatically generated from aarch64_ad.m4
12071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12072 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12073 %{
12074 match(Set dst (SubI src1 (AndI src2 mask)));
12075 ins_cost(INSN_COST);
12076 format %{ "subw $dst, $src1, $src2, uxtb" %}
12077
12078 ins_encode %{
12079 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12080 as_Register($src2$$reg), ext::uxtb);
12081 %}
12082 ins_pipe(ialu_reg_reg);
12083 %}
12084
12085 // This pattern is automatically generated from aarch64_ad.m4
12086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12087 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12088 %{
12089 match(Set dst (SubI src1 (AndI src2 mask)));
12090 ins_cost(INSN_COST);
12091 format %{ "subw $dst, $src1, $src2, uxth" %}
12092
12093 ins_encode %{
12094 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12095 as_Register($src2$$reg), ext::uxth);
12096 %}
12097 ins_pipe(ialu_reg_reg);
12098 %}
12099
12100 // This pattern is automatically generated from aarch64_ad.m4
12101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12102 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12103 %{
12104 match(Set dst (SubL src1 (AndL src2 mask)));
12105 ins_cost(INSN_COST);
12106 format %{ "sub $dst, $src1, $src2, uxtb" %}
12107
12108 ins_encode %{
12109 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12110 as_Register($src2$$reg), ext::uxtb);
12111 %}
12112 ins_pipe(ialu_reg_reg);
12113 %}
12114
12115 // This pattern is automatically generated from aarch64_ad.m4
12116 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12117 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12118 %{
12119 match(Set dst (SubL src1 (AndL src2 mask)));
12120 ins_cost(INSN_COST);
12121 format %{ "sub $dst, $src1, $src2, uxth" %}
12122
12123 ins_encode %{
12124 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12125 as_Register($src2$$reg), ext::uxth);
12126 %}
12127 ins_pipe(ialu_reg_reg);
12128 %}
12129
12130 // This pattern is automatically generated from aarch64_ad.m4
12131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12132 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12133 %{
12134 match(Set dst (SubL src1 (AndL src2 mask)));
12135 ins_cost(INSN_COST);
12136 format %{ "sub $dst, $src1, $src2, uxtw" %}
12137
12138 ins_encode %{
12139 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12140 as_Register($src2$$reg), ext::uxtw);
12141 %}
12142 ins_pipe(ialu_reg_reg);
12143 %}
12144
12145
12146 // This pattern is automatically generated from aarch64_ad.m4
12147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12148 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12149 %{
12150 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12151 ins_cost(1.9 * INSN_COST);
12152 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12153
12154 ins_encode %{
12155 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12156 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12157 %}
12158 ins_pipe(ialu_reg_reg_shift);
12159 %}
12160
12161 // This pattern is automatically generated from aarch64_ad.m4
12162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12163 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12164 %{
12165 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12166 ins_cost(1.9 * INSN_COST);
12167 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12168
12169 ins_encode %{
12170 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12171 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12172 %}
12173 ins_pipe(ialu_reg_reg_shift);
12174 %}
12175
12176 // This pattern is automatically generated from aarch64_ad.m4
12177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12178 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12179 %{
12180 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12181 ins_cost(1.9 * INSN_COST);
12182 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12183
12184 ins_encode %{
12185 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12186 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12187 %}
12188 ins_pipe(ialu_reg_reg_shift);
12189 %}
12190
12191 // This pattern is automatically generated from aarch64_ad.m4
12192 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12193 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12194 %{
12195 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12196 ins_cost(1.9 * INSN_COST);
12197 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12198
12199 ins_encode %{
12200 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12201 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12202 %}
12203 ins_pipe(ialu_reg_reg_shift);
12204 %}
12205
12206 // This pattern is automatically generated from aarch64_ad.m4
12207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12208 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12209 %{
12210 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12211 ins_cost(1.9 * INSN_COST);
12212 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12213
12214 ins_encode %{
12215 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12216 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12217 %}
12218 ins_pipe(ialu_reg_reg_shift);
12219 %}
12220
12221 // This pattern is automatically generated from aarch64_ad.m4
12222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12223 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12224 %{
12225 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12226 ins_cost(1.9 * INSN_COST);
12227 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12228
12229 ins_encode %{
12230 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12231 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12232 %}
12233 ins_pipe(ialu_reg_reg_shift);
12234 %}
12235
12236 // This pattern is automatically generated from aarch64_ad.m4
12237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12238 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12239 %{
12240 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12241 ins_cost(1.9 * INSN_COST);
12242 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12243
12244 ins_encode %{
12245 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12246 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12247 %}
12248 ins_pipe(ialu_reg_reg_shift);
12249 %}
12250
12251 // This pattern is automatically generated from aarch64_ad.m4
12252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12253 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12254 %{
12255 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12256 ins_cost(1.9 * INSN_COST);
12257 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12258
12259 ins_encode %{
12260 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12261 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12262 %}
12263 ins_pipe(ialu_reg_reg_shift);
12264 %}
12265
12266 // This pattern is automatically generated from aarch64_ad.m4
12267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12268 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12269 %{
12270 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12271 ins_cost(1.9 * INSN_COST);
12272 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12273
12274 ins_encode %{
12275 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12276 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12277 %}
12278 ins_pipe(ialu_reg_reg_shift);
12279 %}
12280
12281 // This pattern is automatically generated from aarch64_ad.m4
12282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12283 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12284 %{
12285 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12286 ins_cost(1.9 * INSN_COST);
12287 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12288
12289 ins_encode %{
12290 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12291 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12292 %}
12293 ins_pipe(ialu_reg_reg_shift);
12294 %}
12295
12296 // This pattern is automatically generated from aarch64_ad.m4
12297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12298 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12299 %{
12300 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12301 ins_cost(1.9 * INSN_COST);
12302 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12303
12304 ins_encode %{
12305 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12306 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12307 %}
12308 ins_pipe(ialu_reg_reg_shift);
12309 %}
12310
12311 // This pattern is automatically generated from aarch64_ad.m4
12312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12313 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12314 %{
12315 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12316 ins_cost(1.9 * INSN_COST);
12317 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12318
12319 ins_encode %{
12320 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12321 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12322 %}
12323 ins_pipe(ialu_reg_reg_shift);
12324 %}
12325
12326 // This pattern is automatically generated from aarch64_ad.m4
12327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12328 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12329 %{
12330 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12331 ins_cost(1.9 * INSN_COST);
12332 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12333
12334 ins_encode %{
12335 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12336 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12337 %}
12338 ins_pipe(ialu_reg_reg_shift);
12339 %}
12340
12341 // This pattern is automatically generated from aarch64_ad.m4
12342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12343 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12344 %{
12345 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12346 ins_cost(1.9 * INSN_COST);
12347 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12348
12349 ins_encode %{
12350 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12351 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12352 %}
12353 ins_pipe(ialu_reg_reg_shift);
12354 %}
12355
12356 // This pattern is automatically generated from aarch64_ad.m4
12357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12358 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12359 %{
12360 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12361 ins_cost(1.9 * INSN_COST);
12362 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12363
12364 ins_encode %{
12365 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12366 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12367 %}
12368 ins_pipe(ialu_reg_reg_shift);
12369 %}
12370
12371 // This pattern is automatically generated from aarch64_ad.m4
12372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12373 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12374 %{
12375 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12376 ins_cost(1.9 * INSN_COST);
12377 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12378
12379 ins_encode %{
12380 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12381 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12382 %}
12383 ins_pipe(ialu_reg_reg_shift);
12384 %}
12385
12386 // This pattern is automatically generated from aarch64_ad.m4
12387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12388 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12389 %{
12390 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12391 ins_cost(1.9 * INSN_COST);
12392 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12393
12394 ins_encode %{
12395 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12396 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12397 %}
12398 ins_pipe(ialu_reg_reg_shift);
12399 %}
12400
12401 // This pattern is automatically generated from aarch64_ad.m4
12402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12403 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12404 %{
12405 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12406 ins_cost(1.9 * INSN_COST);
12407 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12408
12409 ins_encode %{
12410 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12411 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12412 %}
12413 ins_pipe(ialu_reg_reg_shift);
12414 %}
12415
12416 // This pattern is automatically generated from aarch64_ad.m4
12417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12418 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12419 %{
12420 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12421 ins_cost(1.9 * INSN_COST);
12422 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12423
12424 ins_encode %{
12425 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12426 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12427 %}
12428 ins_pipe(ialu_reg_reg_shift);
12429 %}
12430
12431 // This pattern is automatically generated from aarch64_ad.m4
12432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12433 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12434 %{
12435 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12436 ins_cost(1.9 * INSN_COST);
12437 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12438
12439 ins_encode %{
12440 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12441 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12442 %}
12443 ins_pipe(ialu_reg_reg_shift);
12444 %}
12445
12446 // This pattern is automatically generated from aarch64_ad.m4
12447 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12448 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12449 %{
12450 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12451 ins_cost(1.9 * INSN_COST);
12452 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12453
12454 ins_encode %{
12455 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12456 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12457 %}
12458 ins_pipe(ialu_reg_reg_shift);
12459 %}
12460
12461 // This pattern is automatically generated from aarch64_ad.m4
12462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12463 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12464 %{
12465 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12466 ins_cost(1.9 * INSN_COST);
12467 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12468
12469 ins_encode %{
12470 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12471 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12472 %}
12473 ins_pipe(ialu_reg_reg_shift);
12474 %}
12475
12476 // This pattern is automatically generated from aarch64_ad.m4
12477 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12478 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12479 %{
12480 effect(DEF dst, USE src1, USE src2, USE cr);
12481 ins_cost(INSN_COST * 2);
12482 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12483
12484 ins_encode %{
12485 __ cselw($dst$$Register,
12486 $src1$$Register,
12487 $src2$$Register,
12488 Assembler::LT);
12489 %}
12490 ins_pipe(icond_reg_reg);
12491 %}
12492
12493 // This pattern is automatically generated from aarch64_ad.m4
12494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12495 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12496 %{
12497 effect(DEF dst, USE src1, USE src2, USE cr);
12498 ins_cost(INSN_COST * 2);
12499 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12500
12501 ins_encode %{
12502 __ cselw($dst$$Register,
12503 $src1$$Register,
12504 $src2$$Register,
12505 Assembler::GT);
12506 %}
12507 ins_pipe(icond_reg_reg);
12508 %}
12509
12510 // This pattern is automatically generated from aarch64_ad.m4
12511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12512 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12513 %{
12514 effect(DEF dst, USE src1, USE cr);
12515 ins_cost(INSN_COST * 2);
12516 format %{ "cselw $dst, $src1, zr lt\t" %}
12517
12518 ins_encode %{
12519 __ cselw($dst$$Register,
12520 $src1$$Register,
12521 zr,
12522 Assembler::LT);
12523 %}
12524 ins_pipe(icond_reg);
12525 %}
12526
12527 // This pattern is automatically generated from aarch64_ad.m4
12528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12529 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12530 %{
12531 effect(DEF dst, USE src1, USE cr);
12532 ins_cost(INSN_COST * 2);
12533 format %{ "cselw $dst, $src1, zr gt\t" %}
12534
12535 ins_encode %{
12536 __ cselw($dst$$Register,
12537 $src1$$Register,
12538 zr,
12539 Assembler::GT);
12540 %}
12541 ins_pipe(icond_reg);
12542 %}
12543
12544 // This pattern is automatically generated from aarch64_ad.m4
12545 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12546 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12547 %{
12548 effect(DEF dst, USE src1, USE cr);
12549 ins_cost(INSN_COST * 2);
12550 format %{ "csincw $dst, $src1, zr le\t" %}
12551
12552 ins_encode %{
12553 __ csincw($dst$$Register,
12554 $src1$$Register,
12555 zr,
12556 Assembler::LE);
12557 %}
12558 ins_pipe(icond_reg);
12559 %}
12560
12561 // This pattern is automatically generated from aarch64_ad.m4
12562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12563 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12564 %{
12565 effect(DEF dst, USE src1, USE cr);
12566 ins_cost(INSN_COST * 2);
12567 format %{ "csincw $dst, $src1, zr gt\t" %}
12568
12569 ins_encode %{
12570 __ csincw($dst$$Register,
12571 $src1$$Register,
12572 zr,
12573 Assembler::GT);
12574 %}
12575 ins_pipe(icond_reg);
12576 %}
12577
12578 // This pattern is automatically generated from aarch64_ad.m4
12579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12580 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12581 %{
12582 effect(DEF dst, USE src1, USE cr);
12583 ins_cost(INSN_COST * 2);
12584 format %{ "csinvw $dst, $src1, zr lt\t" %}
12585
12586 ins_encode %{
12587 __ csinvw($dst$$Register,
12588 $src1$$Register,
12589 zr,
12590 Assembler::LT);
12591 %}
12592 ins_pipe(icond_reg);
12593 %}
12594
12595 // This pattern is automatically generated from aarch64_ad.m4
12596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12597 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12598 %{
12599 effect(DEF dst, USE src1, USE cr);
12600 ins_cost(INSN_COST * 2);
12601 format %{ "csinvw $dst, $src1, zr ge\t" %}
12602
12603 ins_encode %{
12604 __ csinvw($dst$$Register,
12605 $src1$$Register,
12606 zr,
12607 Assembler::GE);
12608 %}
12609 ins_pipe(icond_reg);
12610 %}
12611
12612 // This pattern is automatically generated from aarch64_ad.m4
12613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12614 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12615 %{
12616 match(Set dst (MinI src imm));
12617 ins_cost(INSN_COST * 3);
12618 expand %{
12619 rFlagsReg cr;
12620 compI_reg_imm0(cr, src);
12621 cmovI_reg_imm0_lt(dst, src, cr);
12622 %}
12623 %}
12624
12625 // This pattern is automatically generated from aarch64_ad.m4
12626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12627 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12628 %{
12629 match(Set dst (MinI imm src));
12630 ins_cost(INSN_COST * 3);
12631 expand %{
12632 rFlagsReg cr;
12633 compI_reg_imm0(cr, src);
12634 cmovI_reg_imm0_lt(dst, src, cr);
12635 %}
12636 %}
12637
12638 // This pattern is automatically generated from aarch64_ad.m4
12639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12640 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12641 %{
12642 match(Set dst (MinI src imm));
12643 ins_cost(INSN_COST * 3);
12644 expand %{
12645 rFlagsReg cr;
12646 compI_reg_imm0(cr, src);
12647 cmovI_reg_imm1_le(dst, src, cr);
12648 %}
12649 %}
12650
12651 // This pattern is automatically generated from aarch64_ad.m4
12652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12653 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12654 %{
12655 match(Set dst (MinI imm src));
12656 ins_cost(INSN_COST * 3);
12657 expand %{
12658 rFlagsReg cr;
12659 compI_reg_imm0(cr, src);
12660 cmovI_reg_imm1_le(dst, src, cr);
12661 %}
12662 %}
12663
12664 // This pattern is automatically generated from aarch64_ad.m4
12665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12666 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12667 %{
12668 match(Set dst (MinI src imm));
12669 ins_cost(INSN_COST * 3);
12670 expand %{
12671 rFlagsReg cr;
12672 compI_reg_imm0(cr, src);
12673 cmovI_reg_immM1_lt(dst, src, cr);
12674 %}
12675 %}
12676
12677 // This pattern is automatically generated from aarch64_ad.m4
12678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12679 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12680 %{
12681 match(Set dst (MinI imm src));
12682 ins_cost(INSN_COST * 3);
12683 expand %{
12684 rFlagsReg cr;
12685 compI_reg_imm0(cr, src);
12686 cmovI_reg_immM1_lt(dst, src, cr);
12687 %}
12688 %}
12689
12690 // This pattern is automatically generated from aarch64_ad.m4
12691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12692 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12693 %{
12694 match(Set dst (MaxI src imm));
12695 ins_cost(INSN_COST * 3);
12696 expand %{
12697 rFlagsReg cr;
12698 compI_reg_imm0(cr, src);
12699 cmovI_reg_imm0_gt(dst, src, cr);
12700 %}
12701 %}
12702
12703 // This pattern is automatically generated from aarch64_ad.m4
12704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12705 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12706 %{
12707 match(Set dst (MaxI imm src));
12708 ins_cost(INSN_COST * 3);
12709 expand %{
12710 rFlagsReg cr;
12711 compI_reg_imm0(cr, src);
12712 cmovI_reg_imm0_gt(dst, src, cr);
12713 %}
12714 %}
12715
12716 // This pattern is automatically generated from aarch64_ad.m4
12717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12718 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12719 %{
12720 match(Set dst (MaxI src imm));
12721 ins_cost(INSN_COST * 3);
12722 expand %{
12723 rFlagsReg cr;
12724 compI_reg_imm0(cr, src);
12725 cmovI_reg_imm1_gt(dst, src, cr);
12726 %}
12727 %}
12728
12729 // This pattern is automatically generated from aarch64_ad.m4
12730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12731 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12732 %{
12733 match(Set dst (MaxI imm src));
12734 ins_cost(INSN_COST * 3);
12735 expand %{
12736 rFlagsReg cr;
12737 compI_reg_imm0(cr, src);
12738 cmovI_reg_imm1_gt(dst, src, cr);
12739 %}
12740 %}
12741
12742 // This pattern is automatically generated from aarch64_ad.m4
12743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12744 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12745 %{
12746 match(Set dst (MaxI src imm));
12747 ins_cost(INSN_COST * 3);
12748 expand %{
12749 rFlagsReg cr;
12750 compI_reg_imm0(cr, src);
12751 cmovI_reg_immM1_ge(dst, src, cr);
12752 %}
12753 %}
12754
12755 // This pattern is automatically generated from aarch64_ad.m4
12756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12757 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12758 %{
12759 match(Set dst (MaxI imm src));
12760 ins_cost(INSN_COST * 3);
12761 expand %{
12762 rFlagsReg cr;
12763 compI_reg_imm0(cr, src);
12764 cmovI_reg_immM1_ge(dst, src, cr);
12765 %}
12766 %}
12767
12768 // This pattern is automatically generated from aarch64_ad.m4
12769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12770 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12771 %{
12772 match(Set dst (ReverseI src));
12773 ins_cost(INSN_COST);
12774 format %{ "rbitw $dst, $src" %}
12775 ins_encode %{
12776 __ rbitw($dst$$Register, $src$$Register);
12777 %}
12778 ins_pipe(ialu_reg);
12779 %}
12780
12781 // This pattern is automatically generated from aarch64_ad.m4
12782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12783 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12784 %{
12785 match(Set dst (ReverseL src));
12786 ins_cost(INSN_COST);
12787 format %{ "rbit $dst, $src" %}
12788 ins_encode %{
12789 __ rbit($dst$$Register, $src$$Register);
12790 %}
12791 ins_pipe(ialu_reg);
12792 %}
12793
12794
12795 // END This section of the file is automatically generated. Do not edit --------------
12796
12797
12798 // ============================================================================
12799 // Floating Point Arithmetic Instructions
12800
12801 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12802 match(Set dst (AddHF src1 src2));
12803 format %{ "faddh $dst, $src1, $src2" %}
12804 ins_encode %{
12805 __ faddh($dst$$FloatRegister,
12806 $src1$$FloatRegister,
12807 $src2$$FloatRegister);
12808 %}
12809 ins_pipe(fp_dop_reg_reg_s);
12810 %}
12811
12812 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12813 match(Set dst (AddF src1 src2));
12814
12815 ins_cost(INSN_COST * 5);
12816 format %{ "fadds $dst, $src1, $src2" %}
12817
12818 ins_encode %{
12819 __ fadds(as_FloatRegister($dst$$reg),
12820 as_FloatRegister($src1$$reg),
12821 as_FloatRegister($src2$$reg));
12822 %}
12823
12824 ins_pipe(fp_dop_reg_reg_s);
12825 %}
12826
12827 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12828 match(Set dst (AddD src1 src2));
12829
12830 ins_cost(INSN_COST * 5);
12831 format %{ "faddd $dst, $src1, $src2" %}
12832
12833 ins_encode %{
12834 __ faddd(as_FloatRegister($dst$$reg),
12835 as_FloatRegister($src1$$reg),
12836 as_FloatRegister($src2$$reg));
12837 %}
12838
12839 ins_pipe(fp_dop_reg_reg_d);
12840 %}
12841
12842 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12843 match(Set dst (SubHF src1 src2));
12844 format %{ "fsubh $dst, $src1, $src2" %}
12845 ins_encode %{
12846 __ fsubh($dst$$FloatRegister,
12847 $src1$$FloatRegister,
12848 $src2$$FloatRegister);
12849 %}
12850 ins_pipe(fp_dop_reg_reg_s);
12851 %}
12852
12853 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12854 match(Set dst (SubF src1 src2));
12855
12856 ins_cost(INSN_COST * 5);
12857 format %{ "fsubs $dst, $src1, $src2" %}
12858
12859 ins_encode %{
12860 __ fsubs(as_FloatRegister($dst$$reg),
12861 as_FloatRegister($src1$$reg),
12862 as_FloatRegister($src2$$reg));
12863 %}
12864
12865 ins_pipe(fp_dop_reg_reg_s);
12866 %}
12867
12868 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12869 match(Set dst (SubD src1 src2));
12870
12871 ins_cost(INSN_COST * 5);
12872 format %{ "fsubd $dst, $src1, $src2" %}
12873
12874 ins_encode %{
12875 __ fsubd(as_FloatRegister($dst$$reg),
12876 as_FloatRegister($src1$$reg),
12877 as_FloatRegister($src2$$reg));
12878 %}
12879
12880 ins_pipe(fp_dop_reg_reg_d);
12881 %}
12882
12883 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12884 match(Set dst (MulHF src1 src2));
12885 format %{ "fmulh $dst, $src1, $src2" %}
12886 ins_encode %{
12887 __ fmulh($dst$$FloatRegister,
12888 $src1$$FloatRegister,
12889 $src2$$FloatRegister);
12890 %}
12891 ins_pipe(fp_dop_reg_reg_s);
12892 %}
12893
12894 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12895 match(Set dst (MulF src1 src2));
12896
12897 ins_cost(INSN_COST * 6);
12898 format %{ "fmuls $dst, $src1, $src2" %}
12899
12900 ins_encode %{
12901 __ fmuls(as_FloatRegister($dst$$reg),
12902 as_FloatRegister($src1$$reg),
12903 as_FloatRegister($src2$$reg));
12904 %}
12905
12906 ins_pipe(fp_dop_reg_reg_s);
12907 %}
12908
12909 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12910 match(Set dst (MulD src1 src2));
12911
12912 ins_cost(INSN_COST * 6);
12913 format %{ "fmuld $dst, $src1, $src2" %}
12914
12915 ins_encode %{
12916 __ fmuld(as_FloatRegister($dst$$reg),
12917 as_FloatRegister($src1$$reg),
12918 as_FloatRegister($src2$$reg));
12919 %}
12920
12921 ins_pipe(fp_dop_reg_reg_d);
12922 %}
12923
12924 // src1 * src2 + src3 (half-precision float)
12925 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12926 match(Set dst (FmaHF src3 (Binary src1 src2)));
12927 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12928 ins_encode %{
12929 assert(UseFMA, "Needs FMA instructions support.");
12930 __ fmaddh($dst$$FloatRegister,
12931 $src1$$FloatRegister,
12932 $src2$$FloatRegister,
12933 $src3$$FloatRegister);
12934 %}
12935 ins_pipe(pipe_class_default);
12936 %}
12937
12938 // src1 * src2 + src3
12939 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12940 match(Set dst (FmaF src3 (Binary src1 src2)));
12941
12942 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12943
12944 ins_encode %{
12945 assert(UseFMA, "Needs FMA instructions support.");
12946 __ fmadds(as_FloatRegister($dst$$reg),
12947 as_FloatRegister($src1$$reg),
12948 as_FloatRegister($src2$$reg),
12949 as_FloatRegister($src3$$reg));
12950 %}
12951
12952 ins_pipe(pipe_class_default);
12953 %}
12954
12955 // src1 * src2 + src3
12956 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12957 match(Set dst (FmaD src3 (Binary src1 src2)));
12958
12959 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12960
12961 ins_encode %{
12962 assert(UseFMA, "Needs FMA instructions support.");
12963 __ fmaddd(as_FloatRegister($dst$$reg),
12964 as_FloatRegister($src1$$reg),
12965 as_FloatRegister($src2$$reg),
12966 as_FloatRegister($src3$$reg));
12967 %}
12968
12969 ins_pipe(pipe_class_default);
12970 %}
12971
12972 // src1 * (-src2) + src3
12973 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12974 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12975 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12976
12977 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12978
12979 ins_encode %{
12980 assert(UseFMA, "Needs FMA instructions support.");
12981 __ fmsubs(as_FloatRegister($dst$$reg),
12982 as_FloatRegister($src1$$reg),
12983 as_FloatRegister($src2$$reg),
12984 as_FloatRegister($src3$$reg));
12985 %}
12986
12987 ins_pipe(pipe_class_default);
12988 %}
12989
12990 // src1 * (-src2) + src3
12991 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12992 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12993 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12994
12995 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12996
12997 ins_encode %{
12998 assert(UseFMA, "Needs FMA instructions support.");
12999 __ fmsubd(as_FloatRegister($dst$$reg),
13000 as_FloatRegister($src1$$reg),
13001 as_FloatRegister($src2$$reg),
13002 as_FloatRegister($src3$$reg));
13003 %}
13004
13005 ins_pipe(pipe_class_default);
13006 %}
13007
13008 // src1 * (-src2) - src3
13009 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13010 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13011 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13012
13013 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13014
13015 ins_encode %{
13016 assert(UseFMA, "Needs FMA instructions support.");
13017 __ fnmadds(as_FloatRegister($dst$$reg),
13018 as_FloatRegister($src1$$reg),
13019 as_FloatRegister($src2$$reg),
13020 as_FloatRegister($src3$$reg));
13021 %}
13022
13023 ins_pipe(pipe_class_default);
13024 %}
13025
13026 // src1 * (-src2) - src3
13027 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13028 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13029 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13030
13031 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13032
13033 ins_encode %{
13034 assert(UseFMA, "Needs FMA instructions support.");
13035 __ fnmaddd(as_FloatRegister($dst$$reg),
13036 as_FloatRegister($src1$$reg),
13037 as_FloatRegister($src2$$reg),
13038 as_FloatRegister($src3$$reg));
13039 %}
13040
13041 ins_pipe(pipe_class_default);
13042 %}
13043
13044 // src1 * src2 - src3
13045 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13046 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13047
13048 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13049
13050 ins_encode %{
13051 assert(UseFMA, "Needs FMA instructions support.");
13052 __ fnmsubs(as_FloatRegister($dst$$reg),
13053 as_FloatRegister($src1$$reg),
13054 as_FloatRegister($src2$$reg),
13055 as_FloatRegister($src3$$reg));
13056 %}
13057
13058 ins_pipe(pipe_class_default);
13059 %}
13060
13061 // src1 * src2 - src3
13062 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13063 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13064
13065 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13066
13067 ins_encode %{
13068 assert(UseFMA, "Needs FMA instructions support.");
13069 // n.b. insn name should be fnmsubd
13070 __ fnmsub(as_FloatRegister($dst$$reg),
13071 as_FloatRegister($src1$$reg),
13072 as_FloatRegister($src2$$reg),
13073 as_FloatRegister($src3$$reg));
13074 %}
13075
13076 ins_pipe(pipe_class_default);
13077 %}
13078
13079 // Math.max(HH)H (half-precision float)
13080 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13081 match(Set dst (MaxHF src1 src2));
13082 format %{ "fmaxh $dst, $src1, $src2" %}
13083 ins_encode %{
13084 __ fmaxh($dst$$FloatRegister,
13085 $src1$$FloatRegister,
13086 $src2$$FloatRegister);
13087 %}
13088 ins_pipe(fp_dop_reg_reg_s);
13089 %}
13090
13091 // Math.min(HH)H (half-precision float)
13092 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13093 match(Set dst (MinHF src1 src2));
13094 format %{ "fminh $dst, $src1, $src2" %}
13095 ins_encode %{
13096 __ fminh($dst$$FloatRegister,
13097 $src1$$FloatRegister,
13098 $src2$$FloatRegister);
13099 %}
13100 ins_pipe(fp_dop_reg_reg_s);
13101 %}
13102
13103 // Math.max(FF)F
13104 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13105 match(Set dst (MaxF src1 src2));
13106
13107 format %{ "fmaxs $dst, $src1, $src2" %}
13108 ins_encode %{
13109 __ fmaxs(as_FloatRegister($dst$$reg),
13110 as_FloatRegister($src1$$reg),
13111 as_FloatRegister($src2$$reg));
13112 %}
13113
13114 ins_pipe(fp_dop_reg_reg_s);
13115 %}
13116
13117 // Math.min(FF)F
13118 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13119 match(Set dst (MinF src1 src2));
13120
13121 format %{ "fmins $dst, $src1, $src2" %}
13122 ins_encode %{
13123 __ fmins(as_FloatRegister($dst$$reg),
13124 as_FloatRegister($src1$$reg),
13125 as_FloatRegister($src2$$reg));
13126 %}
13127
13128 ins_pipe(fp_dop_reg_reg_s);
13129 %}
13130
13131 // Math.max(DD)D
13132 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13133 match(Set dst (MaxD src1 src2));
13134
13135 format %{ "fmaxd $dst, $src1, $src2" %}
13136 ins_encode %{
13137 __ fmaxd(as_FloatRegister($dst$$reg),
13138 as_FloatRegister($src1$$reg),
13139 as_FloatRegister($src2$$reg));
13140 %}
13141
13142 ins_pipe(fp_dop_reg_reg_d);
13143 %}
13144
13145 // Math.min(DD)D
13146 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13147 match(Set dst (MinD src1 src2));
13148
13149 format %{ "fmind $dst, $src1, $src2" %}
13150 ins_encode %{
13151 __ fmind(as_FloatRegister($dst$$reg),
13152 as_FloatRegister($src1$$reg),
13153 as_FloatRegister($src2$$reg));
13154 %}
13155
13156 ins_pipe(fp_dop_reg_reg_d);
13157 %}
13158
13159 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13160 match(Set dst (DivHF src1 src2));
13161 format %{ "fdivh $dst, $src1, $src2" %}
13162 ins_encode %{
13163 __ fdivh($dst$$FloatRegister,
13164 $src1$$FloatRegister,
13165 $src2$$FloatRegister);
13166 %}
13167 ins_pipe(fp_div_s);
13168 %}
13169
13170 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13171 match(Set dst (DivF src1 src2));
13172
13173 ins_cost(INSN_COST * 18);
13174 format %{ "fdivs $dst, $src1, $src2" %}
13175
13176 ins_encode %{
13177 __ fdivs(as_FloatRegister($dst$$reg),
13178 as_FloatRegister($src1$$reg),
13179 as_FloatRegister($src2$$reg));
13180 %}
13181
13182 ins_pipe(fp_div_s);
13183 %}
13184
13185 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13186 match(Set dst (DivD src1 src2));
13187
13188 ins_cost(INSN_COST * 32);
13189 format %{ "fdivd $dst, $src1, $src2" %}
13190
13191 ins_encode %{
13192 __ fdivd(as_FloatRegister($dst$$reg),
13193 as_FloatRegister($src1$$reg),
13194 as_FloatRegister($src2$$reg));
13195 %}
13196
13197 ins_pipe(fp_div_d);
13198 %}
13199
13200 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13201 match(Set dst (NegF src));
13202
13203 ins_cost(INSN_COST * 3);
13204 format %{ "fneg $dst, $src" %}
13205
13206 ins_encode %{
13207 __ fnegs(as_FloatRegister($dst$$reg),
13208 as_FloatRegister($src$$reg));
13209 %}
13210
13211 ins_pipe(fp_uop_s);
13212 %}
13213
13214 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13215 match(Set dst (NegD src));
13216
13217 ins_cost(INSN_COST * 3);
13218 format %{ "fnegd $dst, $src" %}
13219
13220 ins_encode %{
13221 __ fnegd(as_FloatRegister($dst$$reg),
13222 as_FloatRegister($src$$reg));
13223 %}
13224
13225 ins_pipe(fp_uop_d);
13226 %}
13227
13228 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13229 %{
13230 match(Set dst (AbsI src));
13231
13232 effect(KILL cr);
13233 ins_cost(INSN_COST * 2);
13234 format %{ "cmpw $src, zr\n\t"
13235 "cnegw $dst, $src, Assembler::LT\t# int abs"
13236 %}
13237
13238 ins_encode %{
13239 __ cmpw(as_Register($src$$reg), zr);
13240 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13241 %}
13242 ins_pipe(pipe_class_default);
13243 %}
13244
13245 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13246 %{
13247 match(Set dst (AbsL src));
13248
13249 effect(KILL cr);
13250 ins_cost(INSN_COST * 2);
13251 format %{ "cmp $src, zr\n\t"
13252 "cneg $dst, $src, Assembler::LT\t# long abs"
13253 %}
13254
13255 ins_encode %{
13256 __ cmp(as_Register($src$$reg), zr);
13257 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13258 %}
13259 ins_pipe(pipe_class_default);
13260 %}
13261
13262 instruct absF_reg(vRegF dst, vRegF src) %{
13263 match(Set dst (AbsF src));
13264
13265 ins_cost(INSN_COST * 3);
13266 format %{ "fabss $dst, $src" %}
13267 ins_encode %{
13268 __ fabss(as_FloatRegister($dst$$reg),
13269 as_FloatRegister($src$$reg));
13270 %}
13271
13272 ins_pipe(fp_uop_s);
13273 %}
13274
13275 instruct absD_reg(vRegD dst, vRegD src) %{
13276 match(Set dst (AbsD src));
13277
13278 ins_cost(INSN_COST * 3);
13279 format %{ "fabsd $dst, $src" %}
13280 ins_encode %{
13281 __ fabsd(as_FloatRegister($dst$$reg),
13282 as_FloatRegister($src$$reg));
13283 %}
13284
13285 ins_pipe(fp_uop_d);
13286 %}
13287
13288 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13289 match(Set dst (AbsF (SubF src1 src2)));
13290
13291 ins_cost(INSN_COST * 3);
13292 format %{ "fabds $dst, $src1, $src2" %}
13293 ins_encode %{
13294 __ fabds(as_FloatRegister($dst$$reg),
13295 as_FloatRegister($src1$$reg),
13296 as_FloatRegister($src2$$reg));
13297 %}
13298
13299 ins_pipe(fp_uop_s);
13300 %}
13301
13302 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13303 match(Set dst (AbsD (SubD src1 src2)));
13304
13305 ins_cost(INSN_COST * 3);
13306 format %{ "fabdd $dst, $src1, $src2" %}
13307 ins_encode %{
13308 __ fabdd(as_FloatRegister($dst$$reg),
13309 as_FloatRegister($src1$$reg),
13310 as_FloatRegister($src2$$reg));
13311 %}
13312
13313 ins_pipe(fp_uop_d);
13314 %}
13315
13316 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13317 match(Set dst (SqrtD src));
13318
13319 ins_cost(INSN_COST * 50);
13320 format %{ "fsqrtd $dst, $src" %}
13321 ins_encode %{
13322 __ fsqrtd(as_FloatRegister($dst$$reg),
13323 as_FloatRegister($src$$reg));
13324 %}
13325
13326 ins_pipe(fp_div_s);
13327 %}
13328
13329 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13330 match(Set dst (SqrtF src));
13331
13332 ins_cost(INSN_COST * 50);
13333 format %{ "fsqrts $dst, $src" %}
13334 ins_encode %{
13335 __ fsqrts(as_FloatRegister($dst$$reg),
13336 as_FloatRegister($src$$reg));
13337 %}
13338
13339 ins_pipe(fp_div_d);
13340 %}
13341
13342 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13343 match(Set dst (SqrtHF src));
13344 format %{ "fsqrth $dst, $src" %}
13345 ins_encode %{
13346 __ fsqrth($dst$$FloatRegister,
13347 $src$$FloatRegister);
13348 %}
13349 ins_pipe(fp_div_s);
13350 %}
13351
13352 // Math.rint, floor, ceil
13353 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13354 match(Set dst (RoundDoubleMode src rmode));
13355 format %{ "frint $dst, $src, $rmode" %}
13356 ins_encode %{
13357 switch ($rmode$$constant) {
13358 case RoundDoubleModeNode::rmode_rint:
13359 __ frintnd(as_FloatRegister($dst$$reg),
13360 as_FloatRegister($src$$reg));
13361 break;
13362 case RoundDoubleModeNode::rmode_floor:
13363 __ frintmd(as_FloatRegister($dst$$reg),
13364 as_FloatRegister($src$$reg));
13365 break;
13366 case RoundDoubleModeNode::rmode_ceil:
13367 __ frintpd(as_FloatRegister($dst$$reg),
13368 as_FloatRegister($src$$reg));
13369 break;
13370 }
13371 %}
13372 ins_pipe(fp_uop_d);
13373 %}
13374
13375 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13376 match(Set dst (CopySignD src1 (Binary src2 zero)));
13377 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13378 format %{ "CopySignD $dst $src1 $src2" %}
13379 ins_encode %{
13380 FloatRegister dst = as_FloatRegister($dst$$reg),
13381 src1 = as_FloatRegister($src1$$reg),
13382 src2 = as_FloatRegister($src2$$reg),
13383 zero = as_FloatRegister($zero$$reg);
13384 __ fnegd(dst, zero);
13385 __ bsl(dst, __ T8B, src2, src1);
13386 %}
13387 ins_pipe(fp_uop_d);
13388 %}
13389
13390 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13391 match(Set dst (CopySignF src1 src2));
13392 effect(TEMP_DEF dst, USE src1, USE src2);
13393 format %{ "CopySignF $dst $src1 $src2" %}
13394 ins_encode %{
13395 FloatRegister dst = as_FloatRegister($dst$$reg),
13396 src1 = as_FloatRegister($src1$$reg),
13397 src2 = as_FloatRegister($src2$$reg);
13398 __ movi(dst, __ T2S, 0x80, 24);
13399 __ bsl(dst, __ T8B, src2, src1);
13400 %}
13401 ins_pipe(fp_uop_d);
13402 %}
13403
13404 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13405 match(Set dst (SignumD src (Binary zero one)));
13406 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13407 format %{ "signumD $dst, $src" %}
13408 ins_encode %{
13409 FloatRegister src = as_FloatRegister($src$$reg),
13410 dst = as_FloatRegister($dst$$reg),
13411 zero = as_FloatRegister($zero$$reg),
13412 one = as_FloatRegister($one$$reg);
13413 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13414 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13415 // Bit selection instruction gets bit from "one" for each enabled bit in
13416 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13417 // NaN the whole "src" will be copied because "dst" is zero. For all other
13418 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13419 // from "src", and all other bits are copied from 1.0.
13420 __ bsl(dst, __ T8B, one, src);
13421 %}
13422 ins_pipe(fp_uop_d);
13423 %}
13424
13425 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13426 match(Set dst (SignumF src (Binary zero one)));
13427 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13428 format %{ "signumF $dst, $src" %}
13429 ins_encode %{
13430 FloatRegister src = as_FloatRegister($src$$reg),
13431 dst = as_FloatRegister($dst$$reg),
13432 zero = as_FloatRegister($zero$$reg),
13433 one = as_FloatRegister($one$$reg);
13434 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13435 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13436 // Bit selection instruction gets bit from "one" for each enabled bit in
13437 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13438 // NaN the whole "src" will be copied because "dst" is zero. For all other
13439 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13440 // from "src", and all other bits are copied from 1.0.
13441 __ bsl(dst, __ T8B, one, src);
13442 %}
13443 ins_pipe(fp_uop_d);
13444 %}
13445
13446 instruct onspinwait() %{
13447 match(OnSpinWait);
13448 ins_cost(INSN_COST);
13449
13450 format %{ "onspinwait" %}
13451
13452 ins_encode %{
13453 __ spin_wait();
13454 %}
13455 ins_pipe(pipe_class_empty);
13456 %}
13457
13458 // ============================================================================
13459 // Logical Instructions
13460
13461 // Integer Logical Instructions
13462
13463 // And Instructions
13464
13465
13466 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13467 match(Set dst (AndI src1 src2));
13468
13469 format %{ "andw $dst, $src1, $src2\t# int" %}
13470
13471 ins_cost(INSN_COST);
13472 ins_encode %{
13473 __ andw(as_Register($dst$$reg),
13474 as_Register($src1$$reg),
13475 as_Register($src2$$reg));
13476 %}
13477
13478 ins_pipe(ialu_reg_reg);
13479 %}
13480
13481 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13482 match(Set dst (AndI src1 src2));
13483
13484 format %{ "andsw $dst, $src1, $src2\t# int" %}
13485
13486 ins_cost(INSN_COST);
13487 ins_encode %{
13488 __ andw(as_Register($dst$$reg),
13489 as_Register($src1$$reg),
13490 (uint64_t)($src2$$constant));
13491 %}
13492
13493 ins_pipe(ialu_reg_imm);
13494 %}
13495
13496 // Or Instructions
13497
13498 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13499 match(Set dst (OrI src1 src2));
13500
13501 format %{ "orrw $dst, $src1, $src2\t# int" %}
13502
13503 ins_cost(INSN_COST);
13504 ins_encode %{
13505 __ orrw(as_Register($dst$$reg),
13506 as_Register($src1$$reg),
13507 as_Register($src2$$reg));
13508 %}
13509
13510 ins_pipe(ialu_reg_reg);
13511 %}
13512
13513 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13514 match(Set dst (OrI src1 src2));
13515
13516 format %{ "orrw $dst, $src1, $src2\t# int" %}
13517
13518 ins_cost(INSN_COST);
13519 ins_encode %{
13520 __ orrw(as_Register($dst$$reg),
13521 as_Register($src1$$reg),
13522 (uint64_t)($src2$$constant));
13523 %}
13524
13525 ins_pipe(ialu_reg_imm);
13526 %}
13527
13528 // Xor Instructions
13529
13530 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13531 match(Set dst (XorI src1 src2));
13532
13533 format %{ "eorw $dst, $src1, $src2\t# int" %}
13534
13535 ins_cost(INSN_COST);
13536 ins_encode %{
13537 __ eorw(as_Register($dst$$reg),
13538 as_Register($src1$$reg),
13539 as_Register($src2$$reg));
13540 %}
13541
13542 ins_pipe(ialu_reg_reg);
13543 %}
13544
13545 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13546 match(Set dst (XorI src1 src2));
13547
13548 format %{ "eorw $dst, $src1, $src2\t# int" %}
13549
13550 ins_cost(INSN_COST);
13551 ins_encode %{
13552 __ eorw(as_Register($dst$$reg),
13553 as_Register($src1$$reg),
13554 (uint64_t)($src2$$constant));
13555 %}
13556
13557 ins_pipe(ialu_reg_imm);
13558 %}
13559
13560 // Long Logical Instructions
13561 // TODO
13562
13563 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13564 match(Set dst (AndL src1 src2));
13565
13566 format %{ "and $dst, $src1, $src2\t# int" %}
13567
13568 ins_cost(INSN_COST);
13569 ins_encode %{
13570 __ andr(as_Register($dst$$reg),
13571 as_Register($src1$$reg),
13572 as_Register($src2$$reg));
13573 %}
13574
13575 ins_pipe(ialu_reg_reg);
13576 %}
13577
13578 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13579 match(Set dst (AndL src1 src2));
13580
13581 format %{ "and $dst, $src1, $src2\t# int" %}
13582
13583 ins_cost(INSN_COST);
13584 ins_encode %{
13585 __ andr(as_Register($dst$$reg),
13586 as_Register($src1$$reg),
13587 (uint64_t)($src2$$constant));
13588 %}
13589
13590 ins_pipe(ialu_reg_imm);
13591 %}
13592
13593 // Or Instructions
13594
13595 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13596 match(Set dst (OrL src1 src2));
13597
13598 format %{ "orr $dst, $src1, $src2\t# int" %}
13599
13600 ins_cost(INSN_COST);
13601 ins_encode %{
13602 __ orr(as_Register($dst$$reg),
13603 as_Register($src1$$reg),
13604 as_Register($src2$$reg));
13605 %}
13606
13607 ins_pipe(ialu_reg_reg);
13608 %}
13609
13610 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13611 match(Set dst (OrL src1 src2));
13612
13613 format %{ "orr $dst, $src1, $src2\t# int" %}
13614
13615 ins_cost(INSN_COST);
13616 ins_encode %{
13617 __ orr(as_Register($dst$$reg),
13618 as_Register($src1$$reg),
13619 (uint64_t)($src2$$constant));
13620 %}
13621
13622 ins_pipe(ialu_reg_imm);
13623 %}
13624
13625 // Xor Instructions
13626
13627 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13628 match(Set dst (XorL src1 src2));
13629
13630 format %{ "eor $dst, $src1, $src2\t# int" %}
13631
13632 ins_cost(INSN_COST);
13633 ins_encode %{
13634 __ eor(as_Register($dst$$reg),
13635 as_Register($src1$$reg),
13636 as_Register($src2$$reg));
13637 %}
13638
13639 ins_pipe(ialu_reg_reg);
13640 %}
13641
13642 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13643 match(Set dst (XorL src1 src2));
13644
13645 ins_cost(INSN_COST);
13646 format %{ "eor $dst, $src1, $src2\t# int" %}
13647
13648 ins_encode %{
13649 __ eor(as_Register($dst$$reg),
13650 as_Register($src1$$reg),
13651 (uint64_t)($src2$$constant));
13652 %}
13653
13654 ins_pipe(ialu_reg_imm);
13655 %}
13656
13657 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13658 %{
13659 match(Set dst (ConvI2L src));
13660
13661 ins_cost(INSN_COST);
13662 format %{ "sxtw $dst, $src\t# i2l" %}
13663 ins_encode %{
13664 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13665 %}
13666 ins_pipe(ialu_reg_shift);
13667 %}
13668
13669 // this pattern occurs in bigmath arithmetic
13670 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13671 %{
13672 match(Set dst (AndL (ConvI2L src) mask));
13673
13674 ins_cost(INSN_COST);
13675 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13676 ins_encode %{
13677 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13678 %}
13679
13680 ins_pipe(ialu_reg_shift);
13681 %}
13682
13683 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13684 match(Set dst (ConvL2I src));
13685
13686 ins_cost(INSN_COST);
13687 format %{ "movw $dst, $src \t// l2i" %}
13688
13689 ins_encode %{
13690 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13691 %}
13692
13693 ins_pipe(ialu_reg);
13694 %}
13695
13696 instruct convD2F_reg(vRegF dst, vRegD src) %{
13697 match(Set dst (ConvD2F src));
13698
13699 ins_cost(INSN_COST * 5);
13700 format %{ "fcvtd $dst, $src \t// d2f" %}
13701
13702 ins_encode %{
13703 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13704 %}
13705
13706 ins_pipe(fp_d2f);
13707 %}
13708
13709 instruct convF2D_reg(vRegD dst, vRegF src) %{
13710 match(Set dst (ConvF2D src));
13711
13712 ins_cost(INSN_COST * 5);
13713 format %{ "fcvts $dst, $src \t// f2d" %}
13714
13715 ins_encode %{
13716 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13717 %}
13718
13719 ins_pipe(fp_f2d);
13720 %}
13721
13722 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13723 match(Set dst (ConvF2I src));
13724
13725 ins_cost(INSN_COST * 5);
13726 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13727
13728 ins_encode %{
13729 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13730 %}
13731
13732 ins_pipe(fp_f2i);
13733 %}
13734
13735 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13736 match(Set dst (ConvF2L src));
13737
13738 ins_cost(INSN_COST * 5);
13739 format %{ "fcvtzs $dst, $src \t// f2l" %}
13740
13741 ins_encode %{
13742 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13743 %}
13744
13745 ins_pipe(fp_f2l);
13746 %}
13747
13748 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13749 match(Set dst (ConvF2HF src));
13750 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13751 "smov $dst, $tmp\t# move result from $tmp to $dst"
13752 %}
13753 effect(TEMP tmp);
13754 ins_encode %{
13755 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13756 %}
13757 ins_pipe(pipe_slow);
13758 %}
13759
13760 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13761 match(Set dst (ConvHF2F src));
13762 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13763 "fcvt $dst, $tmp\t# convert half to single precision"
13764 %}
13765 effect(TEMP tmp);
13766 ins_encode %{
13767 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13768 %}
13769 ins_pipe(pipe_slow);
13770 %}
13771
13772 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13773 match(Set dst (ConvI2F src));
13774
13775 ins_cost(INSN_COST * 5);
13776 format %{ "scvtfws $dst, $src \t// i2f" %}
13777
13778 ins_encode %{
13779 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13780 %}
13781
13782 ins_pipe(fp_i2f);
13783 %}
13784
13785 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13786 match(Set dst (ConvL2F src));
13787
13788 ins_cost(INSN_COST * 5);
13789 format %{ "scvtfs $dst, $src \t// l2f" %}
13790
13791 ins_encode %{
13792 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13793 %}
13794
13795 ins_pipe(fp_l2f);
13796 %}
13797
13798 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13799 match(Set dst (ConvD2I src));
13800
13801 ins_cost(INSN_COST * 5);
13802 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13803
13804 ins_encode %{
13805 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13806 %}
13807
13808 ins_pipe(fp_d2i);
13809 %}
13810
13811 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13812 match(Set dst (ConvD2L src));
13813
13814 ins_cost(INSN_COST * 5);
13815 format %{ "fcvtzd $dst, $src \t// d2l" %}
13816
13817 ins_encode %{
13818 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13819 %}
13820
13821 ins_pipe(fp_d2l);
13822 %}
13823
13824 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13825 match(Set dst (ConvI2D src));
13826
13827 ins_cost(INSN_COST * 5);
13828 format %{ "scvtfwd $dst, $src \t// i2d" %}
13829
13830 ins_encode %{
13831 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13832 %}
13833
13834 ins_pipe(fp_i2d);
13835 %}
13836
13837 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13838 match(Set dst (ConvL2D src));
13839
13840 ins_cost(INSN_COST * 5);
13841 format %{ "scvtfd $dst, $src \t// l2d" %}
13842
13843 ins_encode %{
13844 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13845 %}
13846
13847 ins_pipe(fp_l2d);
13848 %}
13849
13850 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13851 %{
13852 match(Set dst (RoundD src));
13853 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13854 format %{ "java_round_double $dst,$src"%}
13855 ins_encode %{
13856 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13857 as_FloatRegister($ftmp$$reg));
13858 %}
13859 ins_pipe(pipe_slow);
13860 %}
13861
13862 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13863 %{
13864 match(Set dst (RoundF src));
13865 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13866 format %{ "java_round_float $dst,$src"%}
13867 ins_encode %{
13868 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13869 as_FloatRegister($ftmp$$reg));
13870 %}
13871 ins_pipe(pipe_slow);
13872 %}
13873
13874 // stack <-> reg and reg <-> reg shuffles with no conversion
13875
13876 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13877
13878 match(Set dst (MoveF2I src));
13879
13880 effect(DEF dst, USE src);
13881
13882 ins_cost(4 * INSN_COST);
13883
13884 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13885
13886 ins_encode %{
13887 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13888 %}
13889
13890 ins_pipe(iload_reg_reg);
13891
13892 %}
13893
13894 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13895
13896 match(Set dst (MoveI2F src));
13897
13898 effect(DEF dst, USE src);
13899
13900 ins_cost(4 * INSN_COST);
13901
13902 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13903
13904 ins_encode %{
13905 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13906 %}
13907
13908 ins_pipe(pipe_class_memory);
13909
13910 %}
13911
13912 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13913
13914 match(Set dst (MoveD2L src));
13915
13916 effect(DEF dst, USE src);
13917
13918 ins_cost(4 * INSN_COST);
13919
13920 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13921
13922 ins_encode %{
13923 __ ldr($dst$$Register, Address(sp, $src$$disp));
13924 %}
13925
13926 ins_pipe(iload_reg_reg);
13927
13928 %}
13929
13930 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13931
13932 match(Set dst (MoveL2D src));
13933
13934 effect(DEF dst, USE src);
13935
13936 ins_cost(4 * INSN_COST);
13937
13938 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13939
13940 ins_encode %{
13941 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13942 %}
13943
13944 ins_pipe(pipe_class_memory);
13945
13946 %}
13947
13948 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13949
13950 match(Set dst (MoveF2I src));
13951
13952 effect(DEF dst, USE src);
13953
13954 ins_cost(INSN_COST);
13955
13956 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13957
13958 ins_encode %{
13959 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13960 %}
13961
13962 ins_pipe(pipe_class_memory);
13963
13964 %}
13965
13966 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13967
13968 match(Set dst (MoveI2F src));
13969
13970 effect(DEF dst, USE src);
13971
13972 ins_cost(INSN_COST);
13973
13974 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13975
13976 ins_encode %{
13977 __ strw($src$$Register, Address(sp, $dst$$disp));
13978 %}
13979
13980 ins_pipe(istore_reg_reg);
13981
13982 %}
13983
13984 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13985
13986 match(Set dst (MoveD2L src));
13987
13988 effect(DEF dst, USE src);
13989
13990 ins_cost(INSN_COST);
13991
13992 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13993
13994 ins_encode %{
13995 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13996 %}
13997
13998 ins_pipe(pipe_class_memory);
13999
14000 %}
14001
14002 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14003
14004 match(Set dst (MoveL2D src));
14005
14006 effect(DEF dst, USE src);
14007
14008 ins_cost(INSN_COST);
14009
14010 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14011
14012 ins_encode %{
14013 __ str($src$$Register, Address(sp, $dst$$disp));
14014 %}
14015
14016 ins_pipe(istore_reg_reg);
14017
14018 %}
14019
14020 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14021
14022 match(Set dst (MoveF2I src));
14023
14024 effect(DEF dst, USE src);
14025
14026 ins_cost(INSN_COST);
14027
14028 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14029
14030 ins_encode %{
14031 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14032 %}
14033
14034 ins_pipe(fp_f2i);
14035
14036 %}
14037
14038 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14039
14040 match(Set dst (MoveI2F src));
14041
14042 effect(DEF dst, USE src);
14043
14044 ins_cost(INSN_COST);
14045
14046 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14047
14048 ins_encode %{
14049 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14050 %}
14051
14052 ins_pipe(fp_i2f);
14053
14054 %}
14055
14056 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14057
14058 match(Set dst (MoveD2L src));
14059
14060 effect(DEF dst, USE src);
14061
14062 ins_cost(INSN_COST);
14063
14064 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14065
14066 ins_encode %{
14067 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14068 %}
14069
14070 ins_pipe(fp_d2l);
14071
14072 %}
14073
14074 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14075
14076 match(Set dst (MoveL2D src));
14077
14078 effect(DEF dst, USE src);
14079
14080 ins_cost(INSN_COST);
14081
14082 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14083
14084 ins_encode %{
14085 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14086 %}
14087
14088 ins_pipe(fp_l2d);
14089
14090 %}
14091
14092 // ============================================================================
14093 // clearing of an array
14094
14095 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14096 %{
14097 match(Set dummy (ClearArray cnt base));
14098 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14099
14100 ins_cost(4 * INSN_COST);
14101 format %{ "ClearArray $cnt, $base" %}
14102
14103 ins_encode %{
14104 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14105 if (tpc == nullptr) {
14106 ciEnv::current()->record_failure("CodeCache is full");
14107 return;
14108 }
14109 %}
14110
14111 ins_pipe(pipe_class_memory);
14112 %}
14113
14114 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14115 %{
14116 predicate((uint64_t)n->in(2)->get_long()
14117 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14118 match(Set dummy (ClearArray cnt base));
14119 effect(TEMP temp, USE_KILL base, KILL cr);
14120
14121 ins_cost(4 * INSN_COST);
14122 format %{ "ClearArray $cnt, $base" %}
14123
14124 ins_encode %{
14125 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14126 if (tpc == nullptr) {
14127 ciEnv::current()->record_failure("CodeCache is full");
14128 return;
14129 }
14130 %}
14131
14132 ins_pipe(pipe_class_memory);
14133 %}
14134
14135 // ============================================================================
14136 // Overflow Math Instructions
14137
14138 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14139 %{
14140 match(Set cr (OverflowAddI op1 op2));
14141
14142 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14143 ins_cost(INSN_COST);
14144 ins_encode %{
14145 __ cmnw($op1$$Register, $op2$$Register);
14146 %}
14147
14148 ins_pipe(icmp_reg_reg);
14149 %}
14150
14151 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14152 %{
14153 match(Set cr (OverflowAddI op1 op2));
14154
14155 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14156 ins_cost(INSN_COST);
14157 ins_encode %{
14158 __ cmnw($op1$$Register, $op2$$constant);
14159 %}
14160
14161 ins_pipe(icmp_reg_imm);
14162 %}
14163
14164 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14165 %{
14166 match(Set cr (OverflowAddL op1 op2));
14167
14168 format %{ "cmn $op1, $op2\t# overflow check long" %}
14169 ins_cost(INSN_COST);
14170 ins_encode %{
14171 __ cmn($op1$$Register, $op2$$Register);
14172 %}
14173
14174 ins_pipe(icmp_reg_reg);
14175 %}
14176
14177 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14178 %{
14179 match(Set cr (OverflowAddL op1 op2));
14180
14181 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14182 ins_cost(INSN_COST);
14183 ins_encode %{
14184 __ adds(zr, $op1$$Register, $op2$$constant);
14185 %}
14186
14187 ins_pipe(icmp_reg_imm);
14188 %}
14189
14190 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14191 %{
14192 match(Set cr (OverflowSubI op1 op2));
14193
14194 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14195 ins_cost(INSN_COST);
14196 ins_encode %{
14197 __ cmpw($op1$$Register, $op2$$Register);
14198 %}
14199
14200 ins_pipe(icmp_reg_reg);
14201 %}
14202
14203 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14204 %{
14205 match(Set cr (OverflowSubI op1 op2));
14206
14207 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14208 ins_cost(INSN_COST);
14209 ins_encode %{
14210 __ cmpw($op1$$Register, $op2$$constant);
14211 %}
14212
14213 ins_pipe(icmp_reg_imm);
14214 %}
14215
14216 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14217 %{
14218 match(Set cr (OverflowSubL op1 op2));
14219
14220 format %{ "cmp $op1, $op2\t# overflow check long" %}
14221 ins_cost(INSN_COST);
14222 ins_encode %{
14223 __ cmp($op1$$Register, $op2$$Register);
14224 %}
14225
14226 ins_pipe(icmp_reg_reg);
14227 %}
14228
14229 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14230 %{
14231 match(Set cr (OverflowSubL op1 op2));
14232
14233 format %{ "cmp $op1, $op2\t# overflow check long" %}
14234 ins_cost(INSN_COST);
14235 ins_encode %{
14236 __ subs(zr, $op1$$Register, $op2$$constant);
14237 %}
14238
14239 ins_pipe(icmp_reg_imm);
14240 %}
14241
14242 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14243 %{
14244 match(Set cr (OverflowSubI zero op1));
14245
14246 format %{ "cmpw zr, $op1\t# overflow check int" %}
14247 ins_cost(INSN_COST);
14248 ins_encode %{
14249 __ cmpw(zr, $op1$$Register);
14250 %}
14251
14252 ins_pipe(icmp_reg_imm);
14253 %}
14254
14255 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14256 %{
14257 match(Set cr (OverflowSubL zero op1));
14258
14259 format %{ "cmp zr, $op1\t# overflow check long" %}
14260 ins_cost(INSN_COST);
14261 ins_encode %{
14262 __ cmp(zr, $op1$$Register);
14263 %}
14264
14265 ins_pipe(icmp_reg_imm);
14266 %}
14267
14268 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14269 %{
14270 match(Set cr (OverflowMulI op1 op2));
14271
14272 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14273 "cmp rscratch1, rscratch1, sxtw\n\t"
14274 "movw rscratch1, #0x80000000\n\t"
14275 "cselw rscratch1, rscratch1, zr, NE\n\t"
14276 "cmpw rscratch1, #1" %}
14277 ins_cost(5 * INSN_COST);
14278 ins_encode %{
14279 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14280 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14281 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14282 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14283 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14284 %}
14285
14286 ins_pipe(pipe_slow);
14287 %}
14288
14289 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14290 %{
14291 match(If cmp (OverflowMulI op1 op2));
14292 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14293 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14294 effect(USE labl, KILL cr);
14295
14296 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14297 "cmp rscratch1, rscratch1, sxtw\n\t"
14298 "b$cmp $labl" %}
14299 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14300 ins_encode %{
14301 Label* L = $labl$$label;
14302 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14303 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14304 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14305 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14306 %}
14307
14308 ins_pipe(pipe_serial);
14309 %}
14310
14311 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14312 %{
14313 match(Set cr (OverflowMulL op1 op2));
14314
14315 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14316 "smulh rscratch2, $op1, $op2\n\t"
14317 "cmp rscratch2, rscratch1, ASR #63\n\t"
14318 "movw rscratch1, #0x80000000\n\t"
14319 "cselw rscratch1, rscratch1, zr, NE\n\t"
14320 "cmpw rscratch1, #1" %}
14321 ins_cost(6 * INSN_COST);
14322 ins_encode %{
14323 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14324 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14325 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14326 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14327 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14328 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14329 %}
14330
14331 ins_pipe(pipe_slow);
14332 %}
14333
14334 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14335 %{
14336 match(If cmp (OverflowMulL op1 op2));
14337 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14338 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14339 effect(USE labl, KILL cr);
14340
14341 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14342 "smulh rscratch2, $op1, $op2\n\t"
14343 "cmp rscratch2, rscratch1, ASR #63\n\t"
14344 "b$cmp $labl" %}
14345 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14346 ins_encode %{
14347 Label* L = $labl$$label;
14348 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14349 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14350 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14351 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14352 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14353 %}
14354
14355 ins_pipe(pipe_serial);
14356 %}
14357
14358 // ============================================================================
14359 // Compare Instructions
14360
14361 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14362 %{
14363 match(Set cr (CmpI op1 op2));
14364
14365 effect(DEF cr, USE op1, USE op2);
14366
14367 ins_cost(INSN_COST);
14368 format %{ "cmpw $op1, $op2" %}
14369
14370 ins_encode(aarch64_enc_cmpw(op1, op2));
14371
14372 ins_pipe(icmp_reg_reg);
14373 %}
14374
14375 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14376 %{
14377 match(Set cr (CmpI op1 zero));
14378
14379 effect(DEF cr, USE op1);
14380
14381 ins_cost(INSN_COST);
14382 format %{ "cmpw $op1, 0" %}
14383
14384 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14385
14386 ins_pipe(icmp_reg_imm);
14387 %}
14388
14389 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14390 %{
14391 match(Set cr (CmpI op1 op2));
14392
14393 effect(DEF cr, USE op1);
14394
14395 ins_cost(INSN_COST);
14396 format %{ "cmpw $op1, $op2" %}
14397
14398 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14399
14400 ins_pipe(icmp_reg_imm);
14401 %}
14402
14403 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14404 %{
14405 match(Set cr (CmpI op1 op2));
14406
14407 effect(DEF cr, USE op1);
14408
14409 ins_cost(INSN_COST * 2);
14410 format %{ "cmpw $op1, $op2" %}
14411
14412 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14413
14414 ins_pipe(icmp_reg_imm);
14415 %}
14416
14417 // Unsigned compare Instructions; really, same as signed compare
14418 // except it should only be used to feed an If or a CMovI which takes a
14419 // cmpOpU.
14420
14421 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14422 %{
14423 match(Set cr (CmpU op1 op2));
14424
14425 effect(DEF cr, USE op1, USE op2);
14426
14427 ins_cost(INSN_COST);
14428 format %{ "cmpw $op1, $op2\t# unsigned" %}
14429
14430 ins_encode(aarch64_enc_cmpw(op1, op2));
14431
14432 ins_pipe(icmp_reg_reg);
14433 %}
14434
14435 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14436 %{
14437 match(Set cr (CmpU op1 zero));
14438
14439 effect(DEF cr, USE op1);
14440
14441 ins_cost(INSN_COST);
14442 format %{ "cmpw $op1, #0\t# unsigned" %}
14443
14444 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14445
14446 ins_pipe(icmp_reg_imm);
14447 %}
14448
14449 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14450 %{
14451 match(Set cr (CmpU op1 op2));
14452
14453 effect(DEF cr, USE op1);
14454
14455 ins_cost(INSN_COST);
14456 format %{ "cmpw $op1, $op2\t# unsigned" %}
14457
14458 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14459
14460 ins_pipe(icmp_reg_imm);
14461 %}
14462
14463 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14464 %{
14465 match(Set cr (CmpU op1 op2));
14466
14467 effect(DEF cr, USE op1);
14468
14469 ins_cost(INSN_COST * 2);
14470 format %{ "cmpw $op1, $op2\t# unsigned" %}
14471
14472 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14473
14474 ins_pipe(icmp_reg_imm);
14475 %}
14476
14477 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14478 %{
14479 match(Set cr (CmpL op1 op2));
14480
14481 effect(DEF cr, USE op1, USE op2);
14482
14483 ins_cost(INSN_COST);
14484 format %{ "cmp $op1, $op2" %}
14485
14486 ins_encode(aarch64_enc_cmp(op1, op2));
14487
14488 ins_pipe(icmp_reg_reg);
14489 %}
14490
14491 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14492 %{
14493 match(Set cr (CmpL op1 zero));
14494
14495 effect(DEF cr, USE op1);
14496
14497 ins_cost(INSN_COST);
14498 format %{ "tst $op1" %}
14499
14500 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14501
14502 ins_pipe(icmp_reg_imm);
14503 %}
14504
14505 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14506 %{
14507 match(Set cr (CmpL op1 op2));
14508
14509 effect(DEF cr, USE op1);
14510
14511 ins_cost(INSN_COST);
14512 format %{ "cmp $op1, $op2" %}
14513
14514 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14515
14516 ins_pipe(icmp_reg_imm);
14517 %}
14518
14519 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14520 %{
14521 match(Set cr (CmpL op1 op2));
14522
14523 effect(DEF cr, USE op1);
14524
14525 ins_cost(INSN_COST * 2);
14526 format %{ "cmp $op1, $op2" %}
14527
14528 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14529
14530 ins_pipe(icmp_reg_imm);
14531 %}
14532
14533 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14534 %{
14535 match(Set cr (CmpUL op1 op2));
14536
14537 effect(DEF cr, USE op1, USE op2);
14538
14539 ins_cost(INSN_COST);
14540 format %{ "cmp $op1, $op2" %}
14541
14542 ins_encode(aarch64_enc_cmp(op1, op2));
14543
14544 ins_pipe(icmp_reg_reg);
14545 %}
14546
14547 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14548 %{
14549 match(Set cr (CmpUL op1 zero));
14550
14551 effect(DEF cr, USE op1);
14552
14553 ins_cost(INSN_COST);
14554 format %{ "tst $op1" %}
14555
14556 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14557
14558 ins_pipe(icmp_reg_imm);
14559 %}
14560
14561 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14562 %{
14563 match(Set cr (CmpUL op1 op2));
14564
14565 effect(DEF cr, USE op1);
14566
14567 ins_cost(INSN_COST);
14568 format %{ "cmp $op1, $op2" %}
14569
14570 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14571
14572 ins_pipe(icmp_reg_imm);
14573 %}
14574
14575 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14576 %{
14577 match(Set cr (CmpUL op1 op2));
14578
14579 effect(DEF cr, USE op1);
14580
14581 ins_cost(INSN_COST * 2);
14582 format %{ "cmp $op1, $op2" %}
14583
14584 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14585
14586 ins_pipe(icmp_reg_imm);
14587 %}
14588
14589 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14590 %{
14591 match(Set cr (CmpP op1 op2));
14592
14593 effect(DEF cr, USE op1, USE op2);
14594
14595 ins_cost(INSN_COST);
14596 format %{ "cmp $op1, $op2\t // ptr" %}
14597
14598 ins_encode(aarch64_enc_cmpp(op1, op2));
14599
14600 ins_pipe(icmp_reg_reg);
14601 %}
14602
14603 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14604 %{
14605 match(Set cr (CmpN op1 op2));
14606
14607 effect(DEF cr, USE op1, USE op2);
14608
14609 ins_cost(INSN_COST);
14610 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14611
14612 ins_encode(aarch64_enc_cmpn(op1, op2));
14613
14614 ins_pipe(icmp_reg_reg);
14615 %}
14616
14617 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14618 %{
14619 match(Set cr (CmpP op1 zero));
14620
14621 effect(DEF cr, USE op1, USE zero);
14622
14623 ins_cost(INSN_COST);
14624 format %{ "cmp $op1, 0\t // ptr" %}
14625
14626 ins_encode(aarch64_enc_testp(op1));
14627
14628 ins_pipe(icmp_reg_imm);
14629 %}
14630
14631 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14632 %{
14633 match(Set cr (CmpN op1 zero));
14634
14635 effect(DEF cr, USE op1, USE zero);
14636
14637 ins_cost(INSN_COST);
14638 format %{ "cmp $op1, 0\t // compressed ptr" %}
14639
14640 ins_encode(aarch64_enc_testn(op1));
14641
14642 ins_pipe(icmp_reg_imm);
14643 %}
14644
14645 // FP comparisons
14646 //
14647 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14648 // using normal cmpOp. See declaration of rFlagsReg for details.
14649
14650 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14651 %{
14652 match(Set cr (CmpF src1 src2));
14653
14654 ins_cost(3 * INSN_COST);
14655 format %{ "fcmps $src1, $src2" %}
14656
14657 ins_encode %{
14658 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14659 %}
14660
14661 ins_pipe(pipe_class_compare);
14662 %}
14663
14664 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14665 %{
14666 match(Set cr (CmpF src1 src2));
14667
14668 ins_cost(3 * INSN_COST);
14669 format %{ "fcmps $src1, 0.0" %}
14670
14671 ins_encode %{
14672 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14673 %}
14674
14675 ins_pipe(pipe_class_compare);
14676 %}
14677 // FROM HERE
14678
14679 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14680 %{
14681 match(Set cr (CmpD src1 src2));
14682
14683 ins_cost(3 * INSN_COST);
14684 format %{ "fcmpd $src1, $src2" %}
14685
14686 ins_encode %{
14687 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14688 %}
14689
14690 ins_pipe(pipe_class_compare);
14691 %}
14692
14693 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14694 %{
14695 match(Set cr (CmpD src1 src2));
14696
14697 ins_cost(3 * INSN_COST);
14698 format %{ "fcmpd $src1, 0.0" %}
14699
14700 ins_encode %{
14701 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14702 %}
14703
14704 ins_pipe(pipe_class_compare);
14705 %}
14706
14707 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14708 %{
14709 match(Set dst (CmpF3 src1 src2));
14710 effect(KILL cr);
14711
14712 ins_cost(5 * INSN_COST);
14713 format %{ "fcmps $src1, $src2\n\t"
14714 "csinvw($dst, zr, zr, eq\n\t"
14715 "csnegw($dst, $dst, $dst, lt)"
14716 %}
14717
14718 ins_encode %{
14719 Label done;
14720 FloatRegister s1 = as_FloatRegister($src1$$reg);
14721 FloatRegister s2 = as_FloatRegister($src2$$reg);
14722 Register d = as_Register($dst$$reg);
14723 __ fcmps(s1, s2);
14724 // installs 0 if EQ else -1
14725 __ csinvw(d, zr, zr, Assembler::EQ);
14726 // keeps -1 if less or unordered else installs 1
14727 __ csnegw(d, d, d, Assembler::LT);
14728 __ bind(done);
14729 %}
14730
14731 ins_pipe(pipe_class_default);
14732
14733 %}
14734
14735 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14736 %{
14737 match(Set dst (CmpD3 src1 src2));
14738 effect(KILL cr);
14739
14740 ins_cost(5 * INSN_COST);
14741 format %{ "fcmpd $src1, $src2\n\t"
14742 "csinvw($dst, zr, zr, eq\n\t"
14743 "csnegw($dst, $dst, $dst, lt)"
14744 %}
14745
14746 ins_encode %{
14747 Label done;
14748 FloatRegister s1 = as_FloatRegister($src1$$reg);
14749 FloatRegister s2 = as_FloatRegister($src2$$reg);
14750 Register d = as_Register($dst$$reg);
14751 __ fcmpd(s1, s2);
14752 // installs 0 if EQ else -1
14753 __ csinvw(d, zr, zr, Assembler::EQ);
14754 // keeps -1 if less or unordered else installs 1
14755 __ csnegw(d, d, d, Assembler::LT);
14756 __ bind(done);
14757 %}
14758 ins_pipe(pipe_class_default);
14759
14760 %}
14761
14762 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14763 %{
14764 match(Set dst (CmpF3 src1 zero));
14765 effect(KILL cr);
14766
14767 ins_cost(5 * INSN_COST);
14768 format %{ "fcmps $src1, 0.0\n\t"
14769 "csinvw($dst, zr, zr, eq\n\t"
14770 "csnegw($dst, $dst, $dst, lt)"
14771 %}
14772
14773 ins_encode %{
14774 Label done;
14775 FloatRegister s1 = as_FloatRegister($src1$$reg);
14776 Register d = as_Register($dst$$reg);
14777 __ fcmps(s1, 0.0);
14778 // installs 0 if EQ else -1
14779 __ csinvw(d, zr, zr, Assembler::EQ);
14780 // keeps -1 if less or unordered else installs 1
14781 __ csnegw(d, d, d, Assembler::LT);
14782 __ bind(done);
14783 %}
14784
14785 ins_pipe(pipe_class_default);
14786
14787 %}
14788
14789 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14790 %{
14791 match(Set dst (CmpD3 src1 zero));
14792 effect(KILL cr);
14793
14794 ins_cost(5 * INSN_COST);
14795 format %{ "fcmpd $src1, 0.0\n\t"
14796 "csinvw($dst, zr, zr, eq\n\t"
14797 "csnegw($dst, $dst, $dst, lt)"
14798 %}
14799
14800 ins_encode %{
14801 Label done;
14802 FloatRegister s1 = as_FloatRegister($src1$$reg);
14803 Register d = as_Register($dst$$reg);
14804 __ fcmpd(s1, 0.0);
14805 // installs 0 if EQ else -1
14806 __ csinvw(d, zr, zr, Assembler::EQ);
14807 // keeps -1 if less or unordered else installs 1
14808 __ csnegw(d, d, d, Assembler::LT);
14809 __ bind(done);
14810 %}
14811 ins_pipe(pipe_class_default);
14812
14813 %}
14814
14815 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14816 %{
14817 match(Set dst (CmpLTMask p q));
14818 effect(KILL cr);
14819
14820 ins_cost(3 * INSN_COST);
14821
14822 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14823 "csetw $dst, lt\n\t"
14824 "subw $dst, zr, $dst"
14825 %}
14826
14827 ins_encode %{
14828 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14829 __ csetw(as_Register($dst$$reg), Assembler::LT);
14830 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14831 %}
14832
14833 ins_pipe(ialu_reg_reg);
14834 %}
14835
14836 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14837 %{
14838 match(Set dst (CmpLTMask src zero));
14839 effect(KILL cr);
14840
14841 ins_cost(INSN_COST);
14842
14843 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14844
14845 ins_encode %{
14846 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14847 %}
14848
14849 ins_pipe(ialu_reg_shift);
14850 %}
14851
14852 // ============================================================================
14853 // Max and Min
14854
14855 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14856
14857 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14858 %{
14859 effect(DEF cr, USE src);
14860 ins_cost(INSN_COST);
14861 format %{ "cmpw $src, 0" %}
14862
14863 ins_encode %{
14864 __ cmpw($src$$Register, 0);
14865 %}
14866 ins_pipe(icmp_reg_imm);
14867 %}
14868
14869 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14870 %{
14871 match(Set dst (MinI src1 src2));
14872 ins_cost(INSN_COST * 3);
14873
14874 expand %{
14875 rFlagsReg cr;
14876 compI_reg_reg(cr, src1, src2);
14877 cmovI_reg_reg_lt(dst, src1, src2, cr);
14878 %}
14879 %}
14880
14881 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14882 %{
14883 match(Set dst (MaxI src1 src2));
14884 ins_cost(INSN_COST * 3);
14885
14886 expand %{
14887 rFlagsReg cr;
14888 compI_reg_reg(cr, src1, src2);
14889 cmovI_reg_reg_gt(dst, src1, src2, cr);
14890 %}
14891 %}
14892
14893
14894 // ============================================================================
14895 // Branch Instructions
14896
14897 // Direct Branch.
14898 instruct branch(label lbl)
14899 %{
14900 match(Goto);
14901
14902 effect(USE lbl);
14903
14904 ins_cost(BRANCH_COST);
14905 format %{ "b $lbl" %}
14906
14907 ins_encode(aarch64_enc_b(lbl));
14908
14909 ins_pipe(pipe_branch);
14910 %}
14911
14912 // Conditional Near Branch
14913 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14914 %{
14915 // Same match rule as `branchConFar'.
14916 match(If cmp cr);
14917
14918 effect(USE lbl);
14919
14920 ins_cost(BRANCH_COST);
14921 // If set to 1 this indicates that the current instruction is a
14922 // short variant of a long branch. This avoids using this
14923 // instruction in first-pass matching. It will then only be used in
14924 // the `Shorten_branches' pass.
14925 // ins_short_branch(1);
14926 format %{ "b$cmp $lbl" %}
14927
14928 ins_encode(aarch64_enc_br_con(cmp, lbl));
14929
14930 ins_pipe(pipe_branch_cond);
14931 %}
14932
14933 // Conditional Near Branch Unsigned
14934 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14935 %{
14936 // Same match rule as `branchConFar'.
14937 match(If cmp cr);
14938
14939 effect(USE lbl);
14940
14941 ins_cost(BRANCH_COST);
14942 // If set to 1 this indicates that the current instruction is a
14943 // short variant of a long branch. This avoids using this
14944 // instruction in first-pass matching. It will then only be used in
14945 // the `Shorten_branches' pass.
14946 // ins_short_branch(1);
14947 format %{ "b$cmp $lbl\t# unsigned" %}
14948
14949 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14950
14951 ins_pipe(pipe_branch_cond);
14952 %}
14953
14954 // Make use of CBZ and CBNZ. These instructions, as well as being
14955 // shorter than (cmp; branch), have the additional benefit of not
14956 // killing the flags.
14957
14958 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14959 match(If cmp (CmpI op1 op2));
14960 effect(USE labl);
14961
14962 ins_cost(BRANCH_COST);
14963 format %{ "cbw$cmp $op1, $labl" %}
14964 ins_encode %{
14965 Label* L = $labl$$label;
14966 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14967 if (cond == Assembler::EQ)
14968 __ cbzw($op1$$Register, *L);
14969 else
14970 __ cbnzw($op1$$Register, *L);
14971 %}
14972 ins_pipe(pipe_cmp_branch);
14973 %}
14974
14975 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14976 match(If cmp (CmpL op1 op2));
14977 effect(USE labl);
14978
14979 ins_cost(BRANCH_COST);
14980 format %{ "cb$cmp $op1, $labl" %}
14981 ins_encode %{
14982 Label* L = $labl$$label;
14983 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14984 if (cond == Assembler::EQ)
14985 __ cbz($op1$$Register, *L);
14986 else
14987 __ cbnz($op1$$Register, *L);
14988 %}
14989 ins_pipe(pipe_cmp_branch);
14990 %}
14991
14992 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14993 match(If cmp (CmpP op1 op2));
14994 effect(USE labl);
14995
14996 ins_cost(BRANCH_COST);
14997 format %{ "cb$cmp $op1, $labl" %}
14998 ins_encode %{
14999 Label* L = $labl$$label;
15000 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15001 if (cond == Assembler::EQ)
15002 __ cbz($op1$$Register, *L);
15003 else
15004 __ cbnz($op1$$Register, *L);
15005 %}
15006 ins_pipe(pipe_cmp_branch);
15007 %}
15008
15009 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15010 match(If cmp (CmpN op1 op2));
15011 effect(USE labl);
15012
15013 ins_cost(BRANCH_COST);
15014 format %{ "cbw$cmp $op1, $labl" %}
15015 ins_encode %{
15016 Label* L = $labl$$label;
15017 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15018 if (cond == Assembler::EQ)
15019 __ cbzw($op1$$Register, *L);
15020 else
15021 __ cbnzw($op1$$Register, *L);
15022 %}
15023 ins_pipe(pipe_cmp_branch);
15024 %}
15025
15026 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15027 match(If cmp (CmpP (DecodeN oop) zero));
15028 effect(USE labl);
15029
15030 ins_cost(BRANCH_COST);
15031 format %{ "cb$cmp $oop, $labl" %}
15032 ins_encode %{
15033 Label* L = $labl$$label;
15034 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15035 if (cond == Assembler::EQ)
15036 __ cbzw($oop$$Register, *L);
15037 else
15038 __ cbnzw($oop$$Register, *L);
15039 %}
15040 ins_pipe(pipe_cmp_branch);
15041 %}
15042
15043 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15044 match(If cmp (CmpU op1 op2));
15045 effect(USE labl);
15046
15047 ins_cost(BRANCH_COST);
15048 format %{ "cbw$cmp $op1, $labl" %}
15049 ins_encode %{
15050 Label* L = $labl$$label;
15051 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15052 if (cond == Assembler::EQ || cond == Assembler::LS) {
15053 __ cbzw($op1$$Register, *L);
15054 } else {
15055 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15056 __ cbnzw($op1$$Register, *L);
15057 }
15058 %}
15059 ins_pipe(pipe_cmp_branch);
15060 %}
15061
15062 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15063 match(If cmp (CmpUL op1 op2));
15064 effect(USE labl);
15065
15066 ins_cost(BRANCH_COST);
15067 format %{ "cb$cmp $op1, $labl" %}
15068 ins_encode %{
15069 Label* L = $labl$$label;
15070 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15071 if (cond == Assembler::EQ || cond == Assembler::LS) {
15072 __ cbz($op1$$Register, *L);
15073 } else {
15074 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15075 __ cbnz($op1$$Register, *L);
15076 }
15077 %}
15078 ins_pipe(pipe_cmp_branch);
15079 %}
15080
15081 // Test bit and Branch
15082
15083 // Patterns for short (< 32KiB) variants
15084 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15085 match(If cmp (CmpL op1 op2));
15086 effect(USE labl);
15087
15088 ins_cost(BRANCH_COST);
15089 format %{ "cb$cmp $op1, $labl # long" %}
15090 ins_encode %{
15091 Label* L = $labl$$label;
15092 Assembler::Condition cond =
15093 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15094 __ tbr(cond, $op1$$Register, 63, *L);
15095 %}
15096 ins_pipe(pipe_cmp_branch);
15097 ins_short_branch(1);
15098 %}
15099
15100 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15101 match(If cmp (CmpI op1 op2));
15102 effect(USE labl);
15103
15104 ins_cost(BRANCH_COST);
15105 format %{ "cb$cmp $op1, $labl # int" %}
15106 ins_encode %{
15107 Label* L = $labl$$label;
15108 Assembler::Condition cond =
15109 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15110 __ tbr(cond, $op1$$Register, 31, *L);
15111 %}
15112 ins_pipe(pipe_cmp_branch);
15113 ins_short_branch(1);
15114 %}
15115
15116 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15117 match(If cmp (CmpL (AndL op1 op2) op3));
15118 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15119 effect(USE labl);
15120
15121 ins_cost(BRANCH_COST);
15122 format %{ "tb$cmp $op1, $op2, $labl" %}
15123 ins_encode %{
15124 Label* L = $labl$$label;
15125 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15126 int bit = exact_log2_long($op2$$constant);
15127 __ tbr(cond, $op1$$Register, bit, *L);
15128 %}
15129 ins_pipe(pipe_cmp_branch);
15130 ins_short_branch(1);
15131 %}
15132
15133 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15134 match(If cmp (CmpI (AndI op1 op2) op3));
15135 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15136 effect(USE labl);
15137
15138 ins_cost(BRANCH_COST);
15139 format %{ "tb$cmp $op1, $op2, $labl" %}
15140 ins_encode %{
15141 Label* L = $labl$$label;
15142 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15143 int bit = exact_log2((juint)$op2$$constant);
15144 __ tbr(cond, $op1$$Register, bit, *L);
15145 %}
15146 ins_pipe(pipe_cmp_branch);
15147 ins_short_branch(1);
15148 %}
15149
15150 // And far variants
15151 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15152 match(If cmp (CmpL op1 op2));
15153 effect(USE labl);
15154
15155 ins_cost(BRANCH_COST);
15156 format %{ "cb$cmp $op1, $labl # long" %}
15157 ins_encode %{
15158 Label* L = $labl$$label;
15159 Assembler::Condition cond =
15160 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15161 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15162 %}
15163 ins_pipe(pipe_cmp_branch);
15164 %}
15165
15166 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15167 match(If cmp (CmpI op1 op2));
15168 effect(USE labl);
15169
15170 ins_cost(BRANCH_COST);
15171 format %{ "cb$cmp $op1, $labl # int" %}
15172 ins_encode %{
15173 Label* L = $labl$$label;
15174 Assembler::Condition cond =
15175 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15176 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15177 %}
15178 ins_pipe(pipe_cmp_branch);
15179 %}
15180
15181 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15182 match(If cmp (CmpL (AndL op1 op2) op3));
15183 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15184 effect(USE labl);
15185
15186 ins_cost(BRANCH_COST);
15187 format %{ "tb$cmp $op1, $op2, $labl" %}
15188 ins_encode %{
15189 Label* L = $labl$$label;
15190 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15191 int bit = exact_log2_long($op2$$constant);
15192 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15193 %}
15194 ins_pipe(pipe_cmp_branch);
15195 %}
15196
15197 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15198 match(If cmp (CmpI (AndI op1 op2) op3));
15199 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15200 effect(USE labl);
15201
15202 ins_cost(BRANCH_COST);
15203 format %{ "tb$cmp $op1, $op2, $labl" %}
15204 ins_encode %{
15205 Label* L = $labl$$label;
15206 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15207 int bit = exact_log2((juint)$op2$$constant);
15208 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15209 %}
15210 ins_pipe(pipe_cmp_branch);
15211 %}
15212
15213 // Test bits
15214
15215 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15216 match(Set cr (CmpL (AndL op1 op2) op3));
15217 predicate(Assembler::operand_valid_for_logical_immediate
15218 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15219
15220 ins_cost(INSN_COST);
15221 format %{ "tst $op1, $op2 # long" %}
15222 ins_encode %{
15223 __ tst($op1$$Register, $op2$$constant);
15224 %}
15225 ins_pipe(ialu_reg_reg);
15226 %}
15227
15228 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15229 match(Set cr (CmpI (AndI op1 op2) op3));
15230 predicate(Assembler::operand_valid_for_logical_immediate
15231 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15232
15233 ins_cost(INSN_COST);
15234 format %{ "tst $op1, $op2 # int" %}
15235 ins_encode %{
15236 __ tstw($op1$$Register, $op2$$constant);
15237 %}
15238 ins_pipe(ialu_reg_reg);
15239 %}
15240
15241 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15242 match(Set cr (CmpL (AndL op1 op2) op3));
15243
15244 ins_cost(INSN_COST);
15245 format %{ "tst $op1, $op2 # long" %}
15246 ins_encode %{
15247 __ tst($op1$$Register, $op2$$Register);
15248 %}
15249 ins_pipe(ialu_reg_reg);
15250 %}
15251
15252 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15253 match(Set cr (CmpI (AndI op1 op2) op3));
15254
15255 ins_cost(INSN_COST);
15256 format %{ "tstw $op1, $op2 # int" %}
15257 ins_encode %{
15258 __ tstw($op1$$Register, $op2$$Register);
15259 %}
15260 ins_pipe(ialu_reg_reg);
15261 %}
15262
15263
15264 // Conditional Far Branch
15265 // Conditional Far Branch Unsigned
15266 // TODO: fixme
15267
15268 // counted loop end branch near
15269 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15270 %{
15271 match(CountedLoopEnd cmp cr);
15272
15273 effect(USE lbl);
15274
15275 ins_cost(BRANCH_COST);
15276 // short variant.
15277 // ins_short_branch(1);
15278 format %{ "b$cmp $lbl \t// counted loop end" %}
15279
15280 ins_encode(aarch64_enc_br_con(cmp, lbl));
15281
15282 ins_pipe(pipe_branch);
15283 %}
15284
15285 // counted loop end branch far
15286 // TODO: fixme
15287
15288 // ============================================================================
15289 // inlined locking and unlocking
15290
15291 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15292 %{
15293 match(Set cr (FastLock object box));
15294 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15295
15296 ins_cost(5 * INSN_COST);
15297 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15298
15299 ins_encode %{
15300 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15301 %}
15302
15303 ins_pipe(pipe_serial);
15304 %}
15305
15306 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15307 %{
15308 match(Set cr (FastUnlock object box));
15309 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15310
15311 ins_cost(5 * INSN_COST);
15312 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15313
15314 ins_encode %{
15315 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15316 %}
15317
15318 ins_pipe(pipe_serial);
15319 %}
15320
15321 // ============================================================================
15322 // Safepoint Instructions
15323
15324 // TODO
15325 // provide a near and far version of this code
15326
15327 instruct safePoint(rFlagsReg cr, iRegP poll)
15328 %{
15329 match(SafePoint poll);
15330 effect(KILL cr);
15331
15332 format %{
15333 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15334 %}
15335 ins_encode %{
15336 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15337 %}
15338 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15339 %}
15340
15341
15342 // ============================================================================
15343 // Procedure Call/Return Instructions
15344
15345 // Call Java Static Instruction
15346
15347 instruct CallStaticJavaDirect(method meth)
15348 %{
15349 match(CallStaticJava);
15350
15351 effect(USE meth);
15352
15353 ins_cost(CALL_COST);
15354
15355 format %{ "call,static $meth \t// ==> " %}
15356
15357 ins_encode(aarch64_enc_java_static_call(meth),
15358 aarch64_enc_call_epilog);
15359
15360 ins_pipe(pipe_class_call);
15361 %}
15362
15363 // TO HERE
15364
15365 // Call Java Dynamic Instruction
15366 instruct CallDynamicJavaDirect(method meth)
15367 %{
15368 match(CallDynamicJava);
15369
15370 effect(USE meth);
15371
15372 ins_cost(CALL_COST);
15373
15374 format %{ "CALL,dynamic $meth \t// ==> " %}
15375
15376 ins_encode(aarch64_enc_java_dynamic_call(meth),
15377 aarch64_enc_call_epilog);
15378
15379 ins_pipe(pipe_class_call);
15380 %}
15381
15382 // Call Runtime Instruction
15383
15384 instruct CallRuntimeDirect(method meth)
15385 %{
15386 match(CallRuntime);
15387
15388 effect(USE meth);
15389
15390 ins_cost(CALL_COST);
15391
15392 format %{ "CALL, runtime $meth" %}
15393
15394 ins_encode( aarch64_enc_java_to_runtime(meth) );
15395
15396 ins_pipe(pipe_class_call);
15397 %}
15398
15399 // Call Runtime Instruction
15400
15401 instruct CallLeafDirect(method meth)
15402 %{
15403 match(CallLeaf);
15404
15405 effect(USE meth);
15406
15407 ins_cost(CALL_COST);
15408
15409 format %{ "CALL, runtime leaf $meth" %}
15410
15411 ins_encode( aarch64_enc_java_to_runtime(meth) );
15412
15413 ins_pipe(pipe_class_call);
15414 %}
15415
15416 // Call Runtime Instruction without safepoint and with vector arguments
15417 instruct CallLeafDirectVector(method meth)
15418 %{
15419 match(CallLeafVector);
15420
15421 effect(USE meth);
15422
15423 ins_cost(CALL_COST);
15424
15425 format %{ "CALL, runtime leaf vector $meth" %}
15426
15427 ins_encode(aarch64_enc_java_to_runtime(meth));
15428
15429 ins_pipe(pipe_class_call);
15430 %}
15431
15432 // Call Runtime Instruction
15433
15434 instruct CallLeafNoFPDirect(method meth)
15435 %{
15436 match(CallLeafNoFP);
15437
15438 effect(USE meth);
15439
15440 ins_cost(CALL_COST);
15441
15442 format %{ "CALL, runtime leaf nofp $meth" %}
15443
15444 ins_encode( aarch64_enc_java_to_runtime(meth) );
15445
15446 ins_pipe(pipe_class_call);
15447 %}
15448
15449 // Tail Call; Jump from runtime stub to Java code.
15450 // Also known as an 'interprocedural jump'.
15451 // Target of jump will eventually return to caller.
15452 // TailJump below removes the return address.
15453 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15454 // emitted just above the TailCall which has reset rfp to the caller state.
15455 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15456 %{
15457 match(TailCall jump_target method_ptr);
15458
15459 ins_cost(CALL_COST);
15460
15461 format %{ "br $jump_target\t# $method_ptr holds method" %}
15462
15463 ins_encode(aarch64_enc_tail_call(jump_target));
15464
15465 ins_pipe(pipe_class_call);
15466 %}
15467
15468 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15469 %{
15470 match(TailJump jump_target ex_oop);
15471
15472 ins_cost(CALL_COST);
15473
15474 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15475
15476 ins_encode(aarch64_enc_tail_jmp(jump_target));
15477
15478 ins_pipe(pipe_class_call);
15479 %}
15480
15481 // Forward exception.
15482 instruct ForwardExceptionjmp()
15483 %{
15484 match(ForwardException);
15485 ins_cost(CALL_COST);
15486
15487 format %{ "b forward_exception_stub" %}
15488 ins_encode %{
15489 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15490 %}
15491 ins_pipe(pipe_class_call);
15492 %}
15493
15494 // Create exception oop: created by stack-crawling runtime code.
15495 // Created exception is now available to this handler, and is setup
15496 // just prior to jumping to this handler. No code emitted.
15497 // TODO check
15498 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15499 instruct CreateException(iRegP_R0 ex_oop)
15500 %{
15501 match(Set ex_oop (CreateEx));
15502
15503 format %{ " -- \t// exception oop; no code emitted" %}
15504
15505 size(0);
15506
15507 ins_encode( /*empty*/ );
15508
15509 ins_pipe(pipe_class_empty);
15510 %}
15511
15512 // Rethrow exception: The exception oop will come in the first
15513 // argument position. Then JUMP (not call) to the rethrow stub code.
15514 instruct RethrowException() %{
15515 match(Rethrow);
15516 ins_cost(CALL_COST);
15517
15518 format %{ "b rethrow_stub" %}
15519
15520 ins_encode( aarch64_enc_rethrow() );
15521
15522 ins_pipe(pipe_class_call);
15523 %}
15524
15525
15526 // Return Instruction
15527 // epilog node loads ret address into lr as part of frame pop
15528 instruct Ret()
15529 %{
15530 match(Return);
15531
15532 format %{ "ret\t// return register" %}
15533
15534 ins_encode( aarch64_enc_ret() );
15535
15536 ins_pipe(pipe_branch);
15537 %}
15538
15539 // Die now.
15540 instruct ShouldNotReachHere() %{
15541 match(Halt);
15542
15543 ins_cost(CALL_COST);
15544 format %{ "ShouldNotReachHere" %}
15545
15546 ins_encode %{
15547 if (is_reachable()) {
15548 const char* str = __ code_string(_halt_reason);
15549 __ stop(str);
15550 }
15551 %}
15552
15553 ins_pipe(pipe_class_default);
15554 %}
15555
15556 // ============================================================================
15557 // Partial Subtype Check
15558 //
15559 // superklass array for an instance of the superklass. Set a hidden
15560 // internal cache on a hit (cache is checked with exposed code in
15561 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15562 // encoding ALSO sets flags.
15563
15564 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15565 %{
15566 match(Set result (PartialSubtypeCheck sub super));
15567 predicate(!UseSecondarySupersTable);
15568 effect(KILL cr, KILL temp);
15569
15570 ins_cost(20 * INSN_COST); // slightly larger than the next version
15571 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15572
15573 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15574
15575 opcode(0x1); // Force zero of result reg on hit
15576
15577 ins_pipe(pipe_class_memory);
15578 %}
15579
15580 // Two versions of partialSubtypeCheck, both used when we need to
15581 // search for a super class in the secondary supers array. The first
15582 // is used when we don't know _a priori_ the class being searched
15583 // for. The second, far more common, is used when we do know: this is
15584 // used for instanceof, checkcast, and any case where C2 can determine
15585 // it by constant propagation.
15586
15587 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15588 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15589 rFlagsReg cr)
15590 %{
15591 match(Set result (PartialSubtypeCheck sub super));
15592 predicate(UseSecondarySupersTable);
15593 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15594
15595 ins_cost(10 * INSN_COST); // slightly larger than the next version
15596 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15597
15598 ins_encode %{
15599 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15600 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15601 $vtemp$$FloatRegister,
15602 $result$$Register, /*L_success*/nullptr);
15603 %}
15604
15605 ins_pipe(pipe_class_memory);
15606 %}
15607
15608 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15609 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15610 rFlagsReg cr)
15611 %{
15612 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15613 predicate(UseSecondarySupersTable);
15614 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15615
15616 ins_cost(5 * INSN_COST); // smaller than the next version
15617 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15618
15619 ins_encode %{
15620 bool success = false;
15621 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15622 if (InlineSecondarySupersTest) {
15623 success =
15624 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15625 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15626 $vtemp$$FloatRegister,
15627 $result$$Register,
15628 super_klass_slot);
15629 } else {
15630 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15631 success = (call != nullptr);
15632 }
15633 if (!success) {
15634 ciEnv::current()->record_failure("CodeCache is full");
15635 return;
15636 }
15637 %}
15638
15639 ins_pipe(pipe_class_memory);
15640 %}
15641
15642 // Intrisics for String.compareTo()
15643
15644 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15645 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15646 %{
15647 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15648 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15649 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15650
15651 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15652 ins_encode %{
15653 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15654 __ string_compare($str1$$Register, $str2$$Register,
15655 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15656 $tmp1$$Register, $tmp2$$Register,
15657 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15658 %}
15659 ins_pipe(pipe_class_memory);
15660 %}
15661
15662 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15663 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15664 %{
15665 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15666 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15667 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15668
15669 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15670 ins_encode %{
15671 __ string_compare($str1$$Register, $str2$$Register,
15672 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15673 $tmp1$$Register, $tmp2$$Register,
15674 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15675 %}
15676 ins_pipe(pipe_class_memory);
15677 %}
15678
15679 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15680 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15681 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15682 %{
15683 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15684 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15685 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15686 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15687
15688 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15689 ins_encode %{
15690 __ string_compare($str1$$Register, $str2$$Register,
15691 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15692 $tmp1$$Register, $tmp2$$Register,
15693 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15694 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15695 %}
15696 ins_pipe(pipe_class_memory);
15697 %}
15698
15699 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15700 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15701 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15702 %{
15703 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15704 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15705 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15706 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15707
15708 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15709 ins_encode %{
15710 __ string_compare($str1$$Register, $str2$$Register,
15711 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15712 $tmp1$$Register, $tmp2$$Register,
15713 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15714 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15715 %}
15716 ins_pipe(pipe_class_memory);
15717 %}
15718
15719 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15720 // these string_compare variants as NEON register type for convenience so that the prototype of
15721 // string_compare can be shared with all variants.
15722
15723 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15724 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15725 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15726 pRegGov_P1 pgtmp2, rFlagsReg cr)
15727 %{
15728 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15729 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15730 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15731 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15732
15733 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15734 ins_encode %{
15735 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15736 __ string_compare($str1$$Register, $str2$$Register,
15737 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15738 $tmp1$$Register, $tmp2$$Register,
15739 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15740 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15741 StrIntrinsicNode::LL);
15742 %}
15743 ins_pipe(pipe_class_memory);
15744 %}
15745
15746 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15747 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15748 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15749 pRegGov_P1 pgtmp2, rFlagsReg cr)
15750 %{
15751 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15752 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15753 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15754 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15755
15756 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15757 ins_encode %{
15758 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15759 __ string_compare($str1$$Register, $str2$$Register,
15760 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15761 $tmp1$$Register, $tmp2$$Register,
15762 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15763 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15764 StrIntrinsicNode::LU);
15765 %}
15766 ins_pipe(pipe_class_memory);
15767 %}
15768
15769 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15770 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15771 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15772 pRegGov_P1 pgtmp2, rFlagsReg cr)
15773 %{
15774 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15775 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15776 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15777 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15778
15779 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15780 ins_encode %{
15781 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15782 __ string_compare($str1$$Register, $str2$$Register,
15783 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15784 $tmp1$$Register, $tmp2$$Register,
15785 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15786 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15787 StrIntrinsicNode::UL);
15788 %}
15789 ins_pipe(pipe_class_memory);
15790 %}
15791
15792 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15793 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15794 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15795 pRegGov_P1 pgtmp2, rFlagsReg cr)
15796 %{
15797 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15798 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15799 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15800 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15801
15802 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15803 ins_encode %{
15804 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15805 __ string_compare($str1$$Register, $str2$$Register,
15806 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15807 $tmp1$$Register, $tmp2$$Register,
15808 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15809 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15810 StrIntrinsicNode::UU);
15811 %}
15812 ins_pipe(pipe_class_memory);
15813 %}
15814
15815 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15816 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15817 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15818 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15819 %{
15820 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15821 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15822 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15823 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15824 TEMP vtmp0, TEMP vtmp1, KILL cr);
15825 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15826 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15827
15828 ins_encode %{
15829 __ string_indexof($str1$$Register, $str2$$Register,
15830 $cnt1$$Register, $cnt2$$Register,
15831 $tmp1$$Register, $tmp2$$Register,
15832 $tmp3$$Register, $tmp4$$Register,
15833 $tmp5$$Register, $tmp6$$Register,
15834 -1, $result$$Register, StrIntrinsicNode::UU);
15835 %}
15836 ins_pipe(pipe_class_memory);
15837 %}
15838
15839 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15840 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15841 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15842 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15843 %{
15844 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15845 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15846 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15847 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15848 TEMP vtmp0, TEMP vtmp1, KILL cr);
15849 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15850 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15851
15852 ins_encode %{
15853 __ string_indexof($str1$$Register, $str2$$Register,
15854 $cnt1$$Register, $cnt2$$Register,
15855 $tmp1$$Register, $tmp2$$Register,
15856 $tmp3$$Register, $tmp4$$Register,
15857 $tmp5$$Register, $tmp6$$Register,
15858 -1, $result$$Register, StrIntrinsicNode::LL);
15859 %}
15860 ins_pipe(pipe_class_memory);
15861 %}
15862
15863 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15864 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15865 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15866 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15867 %{
15868 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15869 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15870 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15871 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15872 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15873 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15874 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15875
15876 ins_encode %{
15877 __ string_indexof($str1$$Register, $str2$$Register,
15878 $cnt1$$Register, $cnt2$$Register,
15879 $tmp1$$Register, $tmp2$$Register,
15880 $tmp3$$Register, $tmp4$$Register,
15881 $tmp5$$Register, $tmp6$$Register,
15882 -1, $result$$Register, StrIntrinsicNode::UL);
15883 %}
15884 ins_pipe(pipe_class_memory);
15885 %}
15886
15887 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15888 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15889 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15890 %{
15891 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15892 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15893 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15894 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15895 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15896 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15897
15898 ins_encode %{
15899 int icnt2 = (int)$int_cnt2$$constant;
15900 __ string_indexof($str1$$Register, $str2$$Register,
15901 $cnt1$$Register, zr,
15902 $tmp1$$Register, $tmp2$$Register,
15903 $tmp3$$Register, $tmp4$$Register, zr, zr,
15904 icnt2, $result$$Register, StrIntrinsicNode::UU);
15905 %}
15906 ins_pipe(pipe_class_memory);
15907 %}
15908
15909 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15910 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15911 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15912 %{
15913 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15914 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15915 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15916 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15917 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15918 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15919
15920 ins_encode %{
15921 int icnt2 = (int)$int_cnt2$$constant;
15922 __ string_indexof($str1$$Register, $str2$$Register,
15923 $cnt1$$Register, zr,
15924 $tmp1$$Register, $tmp2$$Register,
15925 $tmp3$$Register, $tmp4$$Register, zr, zr,
15926 icnt2, $result$$Register, StrIntrinsicNode::LL);
15927 %}
15928 ins_pipe(pipe_class_memory);
15929 %}
15930
15931 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15932 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15933 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15934 %{
15935 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15936 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15937 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15938 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15939 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15940 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15941
15942 ins_encode %{
15943 int icnt2 = (int)$int_cnt2$$constant;
15944 __ string_indexof($str1$$Register, $str2$$Register,
15945 $cnt1$$Register, zr,
15946 $tmp1$$Register, $tmp2$$Register,
15947 $tmp3$$Register, $tmp4$$Register, zr, zr,
15948 icnt2, $result$$Register, StrIntrinsicNode::UL);
15949 %}
15950 ins_pipe(pipe_class_memory);
15951 %}
15952
15953 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15954 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15955 iRegINoSp tmp3, rFlagsReg cr)
15956 %{
15957 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15958 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15959 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15960 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15961
15962 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15963
15964 ins_encode %{
15965 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15966 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15967 $tmp3$$Register);
15968 %}
15969 ins_pipe(pipe_class_memory);
15970 %}
15971
15972 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15973 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15974 iRegINoSp tmp3, rFlagsReg cr)
15975 %{
15976 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15977 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15978 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15979 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15980
15981 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15982
15983 ins_encode %{
15984 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15985 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15986 $tmp3$$Register);
15987 %}
15988 ins_pipe(pipe_class_memory);
15989 %}
15990
15991 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15992 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15993 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15994 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15995 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15996 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15997 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15998 ins_encode %{
15999 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16000 $result$$Register, $ztmp1$$FloatRegister,
16001 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16002 $ptmp$$PRegister, true /* isL */);
16003 %}
16004 ins_pipe(pipe_class_memory);
16005 %}
16006
16007 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16008 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16009 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16010 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16011 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16012 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16013 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16014 ins_encode %{
16015 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16016 $result$$Register, $ztmp1$$FloatRegister,
16017 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16018 $ptmp$$PRegister, false /* isL */);
16019 %}
16020 ins_pipe(pipe_class_memory);
16021 %}
16022
16023 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16024 iRegI_R0 result, rFlagsReg cr)
16025 %{
16026 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16027 match(Set result (StrEquals (Binary str1 str2) cnt));
16028 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16029
16030 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16031 ins_encode %{
16032 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16033 __ string_equals($str1$$Register, $str2$$Register,
16034 $result$$Register, $cnt$$Register);
16035 %}
16036 ins_pipe(pipe_class_memory);
16037 %}
16038
16039 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16040 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16041 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16042 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16043 iRegP_R10 tmp, rFlagsReg cr)
16044 %{
16045 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16046 match(Set result (AryEq ary1 ary2));
16047 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16048 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16049 TEMP vtmp6, TEMP vtmp7, KILL cr);
16050
16051 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16052 ins_encode %{
16053 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16054 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16055 $result$$Register, $tmp$$Register, 1);
16056 if (tpc == nullptr) {
16057 ciEnv::current()->record_failure("CodeCache is full");
16058 return;
16059 }
16060 %}
16061 ins_pipe(pipe_class_memory);
16062 %}
16063
16064 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16065 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16066 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16067 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16068 iRegP_R10 tmp, rFlagsReg cr)
16069 %{
16070 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16071 match(Set result (AryEq ary1 ary2));
16072 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16073 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16074 TEMP vtmp6, TEMP vtmp7, KILL cr);
16075
16076 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16077 ins_encode %{
16078 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16079 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16080 $result$$Register, $tmp$$Register, 2);
16081 if (tpc == nullptr) {
16082 ciEnv::current()->record_failure("CodeCache is full");
16083 return;
16084 }
16085 %}
16086 ins_pipe(pipe_class_memory);
16087 %}
16088
16089 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16090 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16091 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16092 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16093 %{
16094 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16095 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16096 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16097
16098 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16099 ins_encode %{
16100 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16101 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16102 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16103 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16104 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16105 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16106 (BasicType)$basic_type$$constant);
16107 if (tpc == nullptr) {
16108 ciEnv::current()->record_failure("CodeCache is full");
16109 return;
16110 }
16111 %}
16112 ins_pipe(pipe_class_memory);
16113 %}
16114
16115 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16116 %{
16117 match(Set result (CountPositives ary1 len));
16118 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16119 format %{ "count positives byte[] $ary1,$len -> $result" %}
16120 ins_encode %{
16121 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16122 if (tpc == nullptr) {
16123 ciEnv::current()->record_failure("CodeCache is full");
16124 return;
16125 }
16126 %}
16127 ins_pipe( pipe_slow );
16128 %}
16129
16130 // fast char[] to byte[] compression
16131 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16132 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16133 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16134 iRegI_R0 result, rFlagsReg cr)
16135 %{
16136 match(Set result (StrCompressedCopy src (Binary dst len)));
16137 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16138 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16139
16140 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16141 ins_encode %{
16142 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16143 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16144 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16145 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16146 %}
16147 ins_pipe(pipe_slow);
16148 %}
16149
16150 // fast byte[] to char[] inflation
16151 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16152 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16153 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16154 %{
16155 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16156 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16157 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16158 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16159
16160 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16161 ins_encode %{
16162 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16163 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16164 $vtmp2$$FloatRegister, $tmp$$Register);
16165 if (tpc == nullptr) {
16166 ciEnv::current()->record_failure("CodeCache is full");
16167 return;
16168 }
16169 %}
16170 ins_pipe(pipe_class_memory);
16171 %}
16172
16173 // encode char[] to byte[] in ISO_8859_1
16174 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16175 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16176 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16177 iRegI_R0 result, rFlagsReg cr)
16178 %{
16179 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16180 match(Set result (EncodeISOArray src (Binary dst len)));
16181 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16182 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16183
16184 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16185 ins_encode %{
16186 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16187 $result$$Register, false,
16188 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16189 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16190 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16191 %}
16192 ins_pipe(pipe_class_memory);
16193 %}
16194
16195 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16196 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16197 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16198 iRegI_R0 result, rFlagsReg cr)
16199 %{
16200 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16201 match(Set result (EncodeISOArray src (Binary dst len)));
16202 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16203 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16204
16205 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16206 ins_encode %{
16207 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16208 $result$$Register, true,
16209 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16210 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16211 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16212 %}
16213 ins_pipe(pipe_class_memory);
16214 %}
16215
16216 //----------------------------- CompressBits/ExpandBits ------------------------
16217
16218 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16219 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16220 match(Set dst (CompressBits src mask));
16221 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16222 format %{ "mov $tsrc, $src\n\t"
16223 "mov $tmask, $mask\n\t"
16224 "bext $tdst, $tsrc, $tmask\n\t"
16225 "mov $dst, $tdst"
16226 %}
16227 ins_encode %{
16228 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16229 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16230 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16231 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16232 %}
16233 ins_pipe(pipe_slow);
16234 %}
16235
16236 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16237 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16238 match(Set dst (CompressBits (LoadI mem) mask));
16239 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16240 format %{ "ldrs $tsrc, $mem\n\t"
16241 "ldrs $tmask, $mask\n\t"
16242 "bext $tdst, $tsrc, $tmask\n\t"
16243 "mov $dst, $tdst"
16244 %}
16245 ins_encode %{
16246 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16247 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16248 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16249 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16250 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16251 %}
16252 ins_pipe(pipe_slow);
16253 %}
16254
16255 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16256 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16257 match(Set dst (CompressBits src mask));
16258 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16259 format %{ "mov $tsrc, $src\n\t"
16260 "mov $tmask, $mask\n\t"
16261 "bext $tdst, $tsrc, $tmask\n\t"
16262 "mov $dst, $tdst"
16263 %}
16264 ins_encode %{
16265 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16266 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16267 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16268 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16269 %}
16270 ins_pipe(pipe_slow);
16271 %}
16272
16273 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16274 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16275 match(Set dst (CompressBits (LoadL mem) mask));
16276 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16277 format %{ "ldrd $tsrc, $mem\n\t"
16278 "ldrd $tmask, $mask\n\t"
16279 "bext $tdst, $tsrc, $tmask\n\t"
16280 "mov $dst, $tdst"
16281 %}
16282 ins_encode %{
16283 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16284 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16285 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16286 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16287 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16288 %}
16289 ins_pipe(pipe_slow);
16290 %}
16291
16292 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16293 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16294 match(Set dst (ExpandBits src mask));
16295 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16296 format %{ "mov $tsrc, $src\n\t"
16297 "mov $tmask, $mask\n\t"
16298 "bdep $tdst, $tsrc, $tmask\n\t"
16299 "mov $dst, $tdst"
16300 %}
16301 ins_encode %{
16302 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16303 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16304 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16305 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16306 %}
16307 ins_pipe(pipe_slow);
16308 %}
16309
16310 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16311 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16312 match(Set dst (ExpandBits (LoadI mem) mask));
16313 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16314 format %{ "ldrs $tsrc, $mem\n\t"
16315 "ldrs $tmask, $mask\n\t"
16316 "bdep $tdst, $tsrc, $tmask\n\t"
16317 "mov $dst, $tdst"
16318 %}
16319 ins_encode %{
16320 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16321 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16322 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16323 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16324 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16325 %}
16326 ins_pipe(pipe_slow);
16327 %}
16328
16329 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16330 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16331 match(Set dst (ExpandBits src mask));
16332 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16333 format %{ "mov $tsrc, $src\n\t"
16334 "mov $tmask, $mask\n\t"
16335 "bdep $tdst, $tsrc, $tmask\n\t"
16336 "mov $dst, $tdst"
16337 %}
16338 ins_encode %{
16339 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16340 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16341 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16342 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16343 %}
16344 ins_pipe(pipe_slow);
16345 %}
16346
16347
16348 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16349 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16350 match(Set dst (ExpandBits (LoadL mem) mask));
16351 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16352 format %{ "ldrd $tsrc, $mem\n\t"
16353 "ldrd $tmask, $mask\n\t"
16354 "bdep $tdst, $tsrc, $tmask\n\t"
16355 "mov $dst, $tdst"
16356 %}
16357 ins_encode %{
16358 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16359 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16360 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16361 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16362 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16363 %}
16364 ins_pipe(pipe_slow);
16365 %}
16366
16367 //----------------------------- Reinterpret ----------------------------------
16368 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16369 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16370 match(Set dst (ReinterpretHF2S src));
16371 format %{ "reinterpretHF2S $dst, $src" %}
16372 ins_encode %{
16373 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16374 %}
16375 ins_pipe(pipe_slow);
16376 %}
16377
16378 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16379 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16380 match(Set dst (ReinterpretS2HF src));
16381 format %{ "reinterpretS2HF $dst, $src" %}
16382 ins_encode %{
16383 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16384 %}
16385 ins_pipe(pipe_slow);
16386 %}
16387
16388 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16389 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16390 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16391 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16392 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16393 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16394 // can be omitted in this pattern, resulting in -
16395 // fcvt $dst, $src // Convert float to half-precision float
16396 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16397 %{
16398 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16399 format %{ "convF2HFAndS2HF $dst, $src" %}
16400 ins_encode %{
16401 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16402 %}
16403 ins_pipe(pipe_slow);
16404 %}
16405
16406 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16407 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16408 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16409 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16410 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16411 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16412 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16413 // resulting in -
16414 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16415 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16416 %{
16417 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16418 format %{ "convHF2SAndHF2F $dst, $src" %}
16419 ins_encode %{
16420 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16421 %}
16422 ins_pipe(pipe_slow);
16423 %}
16424
16425 // ============================================================================
16426 // This name is KNOWN by the ADLC and cannot be changed.
16427 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16428 // for this guy.
16429 instruct tlsLoadP(thread_RegP dst)
16430 %{
16431 match(Set dst (ThreadLocal));
16432
16433 ins_cost(0);
16434
16435 format %{ " -- \t// $dst=Thread::current(), empty" %}
16436
16437 size(0);
16438
16439 ins_encode( /*empty*/ );
16440
16441 ins_pipe(pipe_class_empty);
16442 %}
16443
16444 //----------PEEPHOLE RULES-----------------------------------------------------
16445 // These must follow all instruction definitions as they use the names
16446 // defined in the instructions definitions.
16447 //
16448 // peepmatch ( root_instr_name [preceding_instruction]* );
16449 //
16450 // peepconstraint %{
16451 // (instruction_number.operand_name relational_op instruction_number.operand_name
16452 // [, ...] );
16453 // // instruction numbers are zero-based using left to right order in peepmatch
16454 //
16455 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16456 // // provide an instruction_number.operand_name for each operand that appears
16457 // // in the replacement instruction's match rule
16458 //
16459 // ---------VM FLAGS---------------------------------------------------------
16460 //
16461 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16462 //
16463 // Each peephole rule is given an identifying number starting with zero and
16464 // increasing by one in the order seen by the parser. An individual peephole
16465 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16466 // on the command-line.
16467 //
16468 // ---------CURRENT LIMITATIONS----------------------------------------------
16469 //
16470 // Only match adjacent instructions in same basic block
16471 // Only equality constraints
16472 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16473 // Only one replacement instruction
16474 //
16475 // ---------EXAMPLE----------------------------------------------------------
16476 //
16477 // // pertinent parts of existing instructions in architecture description
16478 // instruct movI(iRegINoSp dst, iRegI src)
16479 // %{
16480 // match(Set dst (CopyI src));
16481 // %}
16482 //
16483 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16484 // %{
16485 // match(Set dst (AddI dst src));
16486 // effect(KILL cr);
16487 // %}
16488 //
16489 // // Change (inc mov) to lea
16490 // peephole %{
16491 // // increment preceded by register-register move
16492 // peepmatch ( incI_iReg movI );
16493 // // require that the destination register of the increment
16494 // // match the destination register of the move
16495 // peepconstraint ( 0.dst == 1.dst );
16496 // // construct a replacement instruction that sets
16497 // // the destination to ( move's source register + one )
16498 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16499 // %}
16500 //
16501
16502 // Implementation no longer uses movX instructions since
16503 // machine-independent system no longer uses CopyX nodes.
16504 //
16505 // peephole
16506 // %{
16507 // peepmatch (incI_iReg movI);
16508 // peepconstraint (0.dst == 1.dst);
16509 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16510 // %}
16511
16512 // peephole
16513 // %{
16514 // peepmatch (decI_iReg movI);
16515 // peepconstraint (0.dst == 1.dst);
16516 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16517 // %}
16518
16519 // peephole
16520 // %{
16521 // peepmatch (addI_iReg_imm movI);
16522 // peepconstraint (0.dst == 1.dst);
16523 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16524 // %}
16525
16526 // peephole
16527 // %{
16528 // peepmatch (incL_iReg movL);
16529 // peepconstraint (0.dst == 1.dst);
16530 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16531 // %}
16532
16533 // peephole
16534 // %{
16535 // peepmatch (decL_iReg movL);
16536 // peepconstraint (0.dst == 1.dst);
16537 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16538 // %}
16539
16540 // peephole
16541 // %{
16542 // peepmatch (addL_iReg_imm movL);
16543 // peepconstraint (0.dst == 1.dst);
16544 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16545 // %}
16546
16547 // peephole
16548 // %{
16549 // peepmatch (addP_iReg_imm movP);
16550 // peepconstraint (0.dst == 1.dst);
16551 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16552 // %}
16553
16554 // // Change load of spilled value to only a spill
16555 // instruct storeI(memory mem, iRegI src)
16556 // %{
16557 // match(Set mem (StoreI mem src));
16558 // %}
16559 //
16560 // instruct loadI(iRegINoSp dst, memory mem)
16561 // %{
16562 // match(Set dst (LoadI mem));
16563 // %}
16564 //
16565
16566 //----------SMARTSPILL RULES---------------------------------------------------
16567 // These must follow all instruction definitions as they use the names
16568 // defined in the instructions definitions.
16569
16570 // Local Variables:
16571 // mode: c++
16572 // End: