1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1713
1714 //=============================================================================
1715
1716 #ifndef PRODUCT
1717 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1718 st->print("nop \t# %d bytes pad for loops and calls", _count);
1719 }
1720 #endif
1721
1722 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1723 for (int i = 0; i < _count; i++) {
1724 __ nop();
1725 }
1726 }
1727
1728 uint MachNopNode::size(PhaseRegAlloc*) const {
1729 return _count * NativeInstruction::instruction_size;
1730 }
1731
1732 //=============================================================================
1733 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1734
1735 int ConstantTable::calculate_table_base_offset() const {
1736 return 0; // absolute addressing, no offset
1737 }
1738
1739 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1740 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1741 ShouldNotReachHere();
1742 }
1743
1744 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1745 // Empty encoding
1746 }
1747
1748 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1749 return 0;
1750 }
1751
1752 #ifndef PRODUCT
1753 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1754 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1755 }
1756 #endif
1757
1758 #ifndef PRODUCT
1759 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1760 Compile* C = ra_->C;
1761
1762 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1763
1764 if (C->output()->need_stack_bang(framesize))
1765 st->print("# stack bang size=%d\n\t", framesize);
1766
1767 if (VM_Version::use_rop_protection()) {
1768 st->print("ldr zr, [lr]\n\t");
1769 st->print("paciaz\n\t");
1770 }
1771 if (framesize < ((1 << 9) + 2 * wordSize)) {
1772 st->print("sub sp, sp, #%d\n\t", framesize);
1773 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1774 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1775 } else {
1776 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1777 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1778 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1779 st->print("sub sp, sp, rscratch1");
1780 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1886 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1887 st->print("add sp, sp, rscratch1\n\t");
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
1952 if (reg < slots_of_int_registers) {
1953 return rc_int;
1954 }
1955
1956 // we have 32 float register * 8 halves
1957 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1958 if (reg < slots_of_int_registers + slots_of_float_registers) {
1959 return rc_float;
1960 }
1961
1962 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1963 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1964 return rc_predicate;
1965 }
1966
1967 // Between predicate regs & stack is the flags.
1968 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1969
1970 return rc_stack;
1971 }
1972
1973 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1974 Compile* C = ra_->C;
1975
1976 // Get registers to move.
1977 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1978 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1979 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1980 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1981
1982 enum RC src_hi_rc = rc_class(src_hi);
1983 enum RC src_lo_rc = rc_class(src_lo);
1984 enum RC dst_hi_rc = rc_class(dst_hi);
1985 enum RC dst_lo_rc = rc_class(dst_lo);
1986
1987 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1988
1989 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1990 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1991 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1992 "expected aligned-adjacent pairs");
1993 }
1994
1995 if (src_lo == dst_lo && src_hi == dst_hi) {
1996 return 0; // Self copy, no move.
1997 }
1998
1999 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2000 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2001 int src_offset = ra_->reg2offset(src_lo);
2002 int dst_offset = ra_->reg2offset(dst_lo);
2003
2004 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2005 uint ireg = ideal_reg();
2006 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2007 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2008 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2009 if (ireg == Op_VecA && masm) {
2010 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2011 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2012 // stack->stack
2013 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2016 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2017 sve_vector_reg_size_in_bytes);
2018 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2019 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2020 sve_vector_reg_size_in_bytes);
2021 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2022 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2023 as_FloatRegister(Matcher::_regEncode[src_lo]),
2024 as_FloatRegister(Matcher::_regEncode[src_lo]));
2025 } else {
2026 ShouldNotReachHere();
2027 }
2028 } else if (masm) {
2029 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2030 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2031 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2032 // stack->stack
2033 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2034 if (ireg == Op_VecD) {
2035 __ unspill(rscratch1, true, src_offset);
2036 __ spill(rscratch1, true, dst_offset);
2037 } else {
2038 __ spill_copy128(src_offset, dst_offset);
2039 }
2040 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2041 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 ireg == Op_VecD ? __ T8B : __ T16B,
2043 as_FloatRegister(Matcher::_regEncode[src_lo]));
2044 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2045 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2046 ireg == Op_VecD ? __ D : __ Q,
2047 ra_->reg2offset(dst_lo));
2048 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2049 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 ireg == Op_VecD ? __ D : __ Q,
2051 ra_->reg2offset(src_lo));
2052 } else {
2053 ShouldNotReachHere();
2054 }
2055 }
2056 } else if (masm) {
2057 switch (src_lo_rc) {
2058 case rc_int:
2059 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2060 if (is64) {
2061 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2062 as_Register(Matcher::_regEncode[src_lo]));
2063 } else {
2064 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2065 as_Register(Matcher::_regEncode[src_lo]));
2066 }
2067 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2068 if (is64) {
2069 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2070 as_Register(Matcher::_regEncode[src_lo]));
2071 } else {
2072 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 as_Register(Matcher::_regEncode[src_lo]));
2074 }
2075 } else { // gpr --> stack spill
2076 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2077 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2078 }
2079 break;
2080 case rc_float:
2081 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2082 if (is64) {
2083 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2084 as_FloatRegister(Matcher::_regEncode[src_lo]));
2085 } else {
2086 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2087 as_FloatRegister(Matcher::_regEncode[src_lo]));
2088 }
2089 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2090 if (is64) {
2091 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2092 as_FloatRegister(Matcher::_regEncode[src_lo]));
2093 } else {
2094 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 }
2097 } else { // fpr --> stack spill
2098 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2099 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2100 is64 ? __ D : __ S, dst_offset);
2101 }
2102 break;
2103 case rc_stack:
2104 if (dst_lo_rc == rc_int) { // stack --> gpr load
2105 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2106 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2107 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2108 is64 ? __ D : __ S, src_offset);
2109 } else if (dst_lo_rc == rc_predicate) {
2110 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2111 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2112 } else { // stack --> stack copy
2113 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2114 if (ideal_reg() == Op_RegVectMask) {
2115 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2116 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2117 } else {
2118 __ unspill(rscratch1, is64, src_offset);
2119 __ spill(rscratch1, is64, dst_offset);
2120 }
2121 }
2122 break;
2123 case rc_predicate:
2124 if (dst_lo_rc == rc_predicate) {
2125 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2126 } else if (dst_lo_rc == rc_stack) {
2127 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2128 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2129 } else {
2130 assert(false, "bad src and dst rc_class combination.");
2131 ShouldNotReachHere();
2132 }
2133 break;
2134 default:
2135 assert(false, "bad rc_class for spill");
2136 ShouldNotReachHere();
2137 }
2138 }
2139
2140 if (st) {
2141 st->print("spill ");
2142 if (src_lo_rc == rc_stack) {
2143 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2144 } else {
2145 st->print("%s -> ", Matcher::regName[src_lo]);
2146 }
2147 if (dst_lo_rc == rc_stack) {
2148 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2149 } else {
2150 st->print("%s", Matcher::regName[dst_lo]);
2151 }
2152 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2153 int vsize = 0;
2154 switch (ideal_reg()) {
2155 case Op_VecD:
2156 vsize = 64;
2157 break;
2158 case Op_VecX:
2159 vsize = 128;
2160 break;
2161 case Op_VecA:
2162 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2163 break;
2164 default:
2165 assert(false, "bad register type for spill");
2166 ShouldNotReachHere();
2167 }
2168 st->print("\t# vector spill size = %d", vsize);
2169 } else if (ideal_reg() == Op_RegVectMask) {
2170 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2171 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2172 st->print("\t# predicate spill size = %d", vsize);
2173 } else {
2174 st->print("\t# spill size = %d", is64 ? 64 : 32);
2175 }
2176 }
2177
2178 return 0;
2179
2180 }
2181
2182 #ifndef PRODUCT
2183 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2184 if (!ra_)
2185 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2186 else
2187 implementation(nullptr, ra_, false, st);
2188 }
2189 #endif
2190
2191 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2192 implementation(masm, ra_, false, nullptr);
2193 }
2194
2195 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2196 return MachNode::size(ra_);
2197 }
2198
2199 //=============================================================================
2200
2201 #ifndef PRODUCT
2202 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2203 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2204 int reg = ra_->get_reg_first(this);
2205 st->print("add %s, rsp, #%d]\t# box lock",
2206 Matcher::regName[reg], offset);
2207 }
2208 #endif
2209
2210 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2211 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2212 int reg = ra_->get_encode(this);
2213
2214 // This add will handle any 24-bit signed offset. 24 bits allows an
2215 // 8 megabyte stack frame.
2216 __ add(as_Register(reg), sp, offset);
2217 }
2218
2219 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2220 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2221 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2222
2223 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2224 return NativeInstruction::instruction_size;
2225 } else {
2226 return 2 * NativeInstruction::instruction_size;
2227 }
2228 }
2229
2230 //=============================================================================
2231
2232 #ifndef PRODUCT
2233 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2234 {
2235 st->print_cr("# MachUEPNode");
2236 if (UseCompressedClassPointers) {
2237 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2238 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2239 st->print_cr("\tcmpw rscratch1, r10");
2240 } else {
2241 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2242 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2243 st->print_cr("\tcmp rscratch1, r10");
2244 }
2245 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2246 }
2247 #endif
2248
2249 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2250 {
2251 __ ic_check(InteriorEntryAlignment);
2252 }
2253
2254 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2255 {
2256 return MachNode::size(ra_);
2257 }
2258
2259 // REQUIRED EMIT CODE
2260
2261 //=============================================================================
2262
2263 // Emit deopt handler code.
2264 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2265 {
2266 // Note that the code buffer's insts_mark is always relative to insts.
2267 // That's why we must use the macroassembler to generate a handler.
2268 address base = __ start_a_stub(size_deopt_handler());
2269 if (base == nullptr) {
2270 ciEnv::current()->record_failure("CodeCache is full");
2271 return 0; // CodeBuffer::expand failed
2272 }
2273
2274 int offset = __ offset();
2275 Label start;
2276 __ bind(start);
2277 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2278
2279 int entry_offset = __ offset();
2280 __ b(start);
2281
2282 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2283 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2284 "out of bounds read in post-call NOP check");
2285 __ end_a_stub();
2286 return entry_offset;
2287 }
2288
2289 // REQUIRED MATCHER CODE
2290
2291 //=============================================================================
2292
2293 bool Matcher::match_rule_supported(int opcode) {
2294 if (!has_match_rule(opcode))
2295 return false;
2296
2297 switch (opcode) {
2298 case Op_OnSpinWait:
2299 return VM_Version::supports_on_spin_wait();
2300 case Op_CacheWB:
2301 case Op_CacheWBPreSync:
2302 case Op_CacheWBPostSync:
2303 if (!VM_Version::supports_data_cache_line_flush()) {
2304 return false;
2305 }
2306 break;
2307 case Op_ExpandBits:
2308 case Op_CompressBits:
2309 if (!VM_Version::supports_svebitperm()) {
2310 return false;
2311 }
2312 break;
2313 case Op_FmaF:
2314 case Op_FmaD:
2315 case Op_FmaVF:
2316 case Op_FmaVD:
2317 if (!UseFMA) {
2318 return false;
2319 }
2320 break;
2321 case Op_FmaHF:
2322 // UseFMA flag also needs to be checked along with FEAT_FP16
2323 if (!UseFMA || !is_feat_fp16_supported()) {
2324 return false;
2325 }
2326 break;
2327 case Op_AddHF:
2328 case Op_SubHF:
2329 case Op_MulHF:
2330 case Op_DivHF:
2331 case Op_MinHF:
2332 case Op_MaxHF:
2333 case Op_SqrtHF:
2334 // Half-precision floating point scalar operations require FEAT_FP16
2335 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2336 // features are supported.
2337 if (!is_feat_fp16_supported()) {
2338 return false;
2339 }
2340 break;
2341 }
2342
2343 return true; // Per default match rules are supported.
2344 }
2345
2346 const RegMask* Matcher::predicate_reg_mask(void) {
2347 return &_PR_REG_mask;
2348 }
2349
2350 bool Matcher::supports_vector_calling_convention(void) {
2351 return EnableVectorSupport;
2352 }
2353
2354 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2355 assert(EnableVectorSupport, "sanity");
2356 int lo = V0_num;
2357 int hi = V0_H_num;
2358 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2359 hi = V0_K_num;
2360 }
2361 return OptoRegPair(hi, lo);
2362 }
2363
2364 // Is this branch offset short enough that a short branch can be used?
2365 //
2366 // NOTE: If the platform does not provide any short branch variants, then
2367 // this method should return false for offset 0.
2368 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2369 // The passed offset is relative to address of the branch.
2370
2371 return (-32768 <= offset && offset < 32768);
2372 }
2373
2374 // Vector width in bytes.
2375 int Matcher::vector_width_in_bytes(BasicType bt) {
2376 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2377 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2378 // Minimum 2 values in vector
2379 if (size < 2*type2aelembytes(bt)) size = 0;
2380 // But never < 4
2381 if (size < 4) size = 0;
2382 return size;
2383 }
2384
2385 // Limits on vector size (number of elements) loaded into vector.
2386 int Matcher::max_vector_size(const BasicType bt) {
2387 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2388 }
2389
2390 int Matcher::min_vector_size(const BasicType bt) {
2391 // Usually, the shortest vector length supported by AArch64 ISA and
2392 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2393 // vectors in a few special cases.
2394 int size;
2395 switch(bt) {
2396 case T_BOOLEAN:
2397 // Load/store a vector mask with only 2 elements for vector types
2398 // such as "2I/2F/2L/2D".
2399 size = 2;
2400 break;
2401 case T_BYTE:
2402 // Generate a "4B" vector, to support vector cast between "8B/16B"
2403 // and "4S/4I/4L/4F/4D".
2404 size = 4;
2405 break;
2406 case T_SHORT:
2407 // Generate a "2S" vector, to support vector cast between "4S/8S"
2408 // and "2I/2L/2F/2D".
2409 size = 2;
2410 break;
2411 default:
2412 // Limit the min vector length to 64-bit.
2413 size = 8 / type2aelembytes(bt);
2414 // The number of elements in a vector should be at least 2.
2415 size = MAX2(size, 2);
2416 }
2417
2418 int max_size = max_vector_size(bt);
2419 return MIN2(size, max_size);
2420 }
2421
2422 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2423 return Matcher::max_vector_size(bt);
2424 }
2425
2426 // Actual max scalable vector register length.
2427 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2428 return Matcher::max_vector_size(bt);
2429 }
2430
2431 // Vector ideal reg.
2432 uint Matcher::vector_ideal_reg(int len) {
2433 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2434 return Op_VecA;
2435 }
2436 switch(len) {
2437 // For 16-bit/32-bit mask vector, reuse VecD.
2438 case 2:
2439 case 4:
2440 case 8: return Op_VecD;
2441 case 16: return Op_VecX;
2442 }
2443 ShouldNotReachHere();
2444 return 0;
2445 }
2446
2447 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2448 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2449 switch (ideal_reg) {
2450 case Op_VecA: return new vecAOper();
2451 case Op_VecD: return new vecDOper();
2452 case Op_VecX: return new vecXOper();
2453 }
2454 ShouldNotReachHere();
2455 return nullptr;
2456 }
2457
2458 bool Matcher::is_reg2reg_move(MachNode* m) {
2459 return false;
2460 }
2461
2462 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2463 return false;
2464 }
2465
2466 bool Matcher::is_generic_vector(MachOper* opnd) {
2467 return opnd->opcode() == VREG;
2468 }
2469
2470 #ifdef ASSERT
2471 // Return whether or not this register is ever used as an argument.
2472 bool Matcher::can_be_java_arg(int reg)
2473 {
2474 return
2475 reg == R0_num || reg == R0_H_num ||
2476 reg == R1_num || reg == R1_H_num ||
2477 reg == R2_num || reg == R2_H_num ||
2478 reg == R3_num || reg == R3_H_num ||
2479 reg == R4_num || reg == R4_H_num ||
2480 reg == R5_num || reg == R5_H_num ||
2481 reg == R6_num || reg == R6_H_num ||
2482 reg == R7_num || reg == R7_H_num ||
2483 reg == V0_num || reg == V0_H_num ||
2484 reg == V1_num || reg == V1_H_num ||
2485 reg == V2_num || reg == V2_H_num ||
2486 reg == V3_num || reg == V3_H_num ||
2487 reg == V4_num || reg == V4_H_num ||
2488 reg == V5_num || reg == V5_H_num ||
2489 reg == V6_num || reg == V6_H_num ||
2490 reg == V7_num || reg == V7_H_num;
2491 }
2492 #endif
2493
2494 uint Matcher::int_pressure_limit()
2495 {
2496 // JDK-8183543: When taking the number of available registers as int
2497 // register pressure threshold, the jtreg test:
2498 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2499 // failed due to C2 compilation failure with
2500 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2501 //
2502 // A derived pointer is live at CallNode and then is flagged by RA
2503 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2504 // derived pointers and lastly fail to spill after reaching maximum
2505 // number of iterations. Lowering the default pressure threshold to
2506 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2507 // a high register pressure area of the code so that split_DEF can
2508 // generate DefinitionSpillCopy for the derived pointer.
2509 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2510 if (!PreserveFramePointer) {
2511 // When PreserveFramePointer is off, frame pointer is allocatable,
2512 // but different from other SOC registers, it is excluded from
2513 // fatproj's mask because its save type is No-Save. Decrease 1 to
2514 // ensure high pressure at fatproj when PreserveFramePointer is off.
2515 // See check_pressure_at_fatproj().
2516 default_int_pressure_threshold--;
2517 }
2518 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2519 }
2520
2521 uint Matcher::float_pressure_limit()
2522 {
2523 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2524 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2525 }
2526
2527 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2528 return false;
2529 }
2530
2531 const RegMask& Matcher::divI_proj_mask() {
2532 ShouldNotReachHere();
2533 return RegMask::EMPTY;
2534 }
2535
2536 // Register for MODI projection of divmodI.
2537 const RegMask& Matcher::modI_proj_mask() {
2538 ShouldNotReachHere();
2539 return RegMask::EMPTY;
2540 }
2541
2542 // Register for DIVL projection of divmodL.
2543 const RegMask& Matcher::divL_proj_mask() {
2544 ShouldNotReachHere();
2545 return RegMask::EMPTY;
2546 }
2547
2548 // Register for MODL projection of divmodL.
2549 const RegMask& Matcher::modL_proj_mask() {
2550 ShouldNotReachHere();
2551 return RegMask::EMPTY;
2552 }
2553
2554 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2555 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2556 Node* u = addp->fast_out(i);
2557 if (u->is_LoadStore()) {
2558 // On AArch64, LoadStoreNodes (i.e. compare and swap
2559 // instructions) only take register indirect as an operand, so
2560 // any attempt to use an AddPNode as an input to a LoadStoreNode
2561 // must fail.
2562 return false;
2563 }
2564 if (u->is_Mem()) {
2565 int opsize = u->as_Mem()->memory_size();
2566 assert(opsize > 0, "unexpected memory operand size");
2567 if (u->as_Mem()->memory_size() != (1<<shift)) {
2568 return false;
2569 }
2570 }
2571 }
2572 return true;
2573 }
2574
2575 // Convert BoolTest condition to Assembler condition.
2576 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2577 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2578 Assembler::Condition result;
2579 switch(cond) {
2580 case BoolTest::eq:
2581 result = Assembler::EQ; break;
2582 case BoolTest::ne:
2583 result = Assembler::NE; break;
2584 case BoolTest::le:
2585 result = Assembler::LE; break;
2586 case BoolTest::ge:
2587 result = Assembler::GE; break;
2588 case BoolTest::lt:
2589 result = Assembler::LT; break;
2590 case BoolTest::gt:
2591 result = Assembler::GT; break;
2592 case BoolTest::ule:
2593 result = Assembler::LS; break;
2594 case BoolTest::uge:
2595 result = Assembler::HS; break;
2596 case BoolTest::ult:
2597 result = Assembler::LO; break;
2598 case BoolTest::ugt:
2599 result = Assembler::HI; break;
2600 case BoolTest::overflow:
2601 result = Assembler::VS; break;
2602 case BoolTest::no_overflow:
2603 result = Assembler::VC; break;
2604 default:
2605 ShouldNotReachHere();
2606 return Assembler::Condition(-1);
2607 }
2608
2609 // Check conversion
2610 if (cond & BoolTest::unsigned_compare) {
2611 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2612 } else {
2613 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2614 }
2615
2616 return result;
2617 }
2618
2619 // Binary src (Replicate con)
2620 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2621 if (n == nullptr || m == nullptr) {
2622 return false;
2623 }
2624
2625 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2626 return false;
2627 }
2628
2629 Node* imm_node = m->in(1);
2630 if (!imm_node->is_Con()) {
2631 return false;
2632 }
2633
2634 const Type* t = imm_node->bottom_type();
2635 if (!(t->isa_int() || t->isa_long())) {
2636 return false;
2637 }
2638
2639 switch (n->Opcode()) {
2640 case Op_AndV:
2641 case Op_OrV:
2642 case Op_XorV: {
2643 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2644 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2645 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2646 }
2647 case Op_AddVB:
2648 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2649 case Op_AddVS:
2650 case Op_AddVI:
2651 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2652 case Op_AddVL:
2653 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2654 default:
2655 return false;
2656 }
2657 }
2658
2659 // (XorV src (Replicate m1))
2660 // (XorVMask src (MaskAll m1))
2661 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2662 if (n != nullptr && m != nullptr) {
2663 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2664 VectorNode::is_all_ones_vector(m);
2665 }
2666 return false;
2667 }
2668
2669 // Should the matcher clone input 'm' of node 'n'?
2670 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2671 if (is_vshift_con_pattern(n, m) ||
2672 is_vector_bitwise_not_pattern(n, m) ||
2673 is_valid_sve_arith_imm_pattern(n, m) ||
2674 is_encode_and_store_pattern(n, m)) {
2675 mstack.push(m, Visit);
2676 return true;
2677 }
2678 return false;
2679 }
2680
2681 // Should the Matcher clone shifts on addressing modes, expecting them
2682 // to be subsumed into complex addressing expressions or compute them
2683 // into registers?
2684 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2685
2686 // Loads and stores with indirect memory input (e.g., volatile loads and
2687 // stores) do not subsume the input into complex addressing expressions. If
2688 // the addressing expression is input to at least one such load or store, do
2689 // not clone the addressing expression. Query needs_acquiring_load and
2690 // needs_releasing_store as a proxy for indirect memory input, as it is not
2691 // possible to directly query for indirect memory input at this stage.
2692 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2693 Node* n = m->fast_out(i);
2694 if (n->is_Load() && needs_acquiring_load(n)) {
2695 return false;
2696 }
2697 if (n->is_Store() && needs_releasing_store(n)) {
2698 return false;
2699 }
2700 }
2701
2702 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2703 return true;
2704 }
2705
2706 Node *off = m->in(AddPNode::Offset);
2707 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2708 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2709 // Are there other uses besides address expressions?
2710 !is_visited(off)) {
2711 address_visited.set(off->_idx); // Flag as address_visited
2712 mstack.push(off->in(2), Visit);
2713 Node *conv = off->in(1);
2714 if (conv->Opcode() == Op_ConvI2L &&
2715 // Are there other uses besides address expressions?
2716 !is_visited(conv)) {
2717 address_visited.set(conv->_idx); // Flag as address_visited
2718 mstack.push(conv->in(1), Pre_Visit);
2719 } else {
2720 mstack.push(conv, Pre_Visit);
2721 }
2722 address_visited.test_set(m->_idx); // Flag as address_visited
2723 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2724 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2725 return true;
2726 } else if (off->Opcode() == Op_ConvI2L &&
2727 // Are there other uses besides address expressions?
2728 !is_visited(off)) {
2729 address_visited.test_set(m->_idx); // Flag as address_visited
2730 address_visited.set(off->_idx); // Flag as address_visited
2731 mstack.push(off->in(1), Pre_Visit);
2732 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2733 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2734 return true;
2735 }
2736 return false;
2737 }
2738
2739 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2740 { \
2741 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2742 guarantee(DISP == 0, "mode not permitted for volatile"); \
2743 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2744 __ INSN(REG, as_Register(BASE)); \
2745 }
2746
2747
2748 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2749 {
2750 Address::extend scale;
2751
2752 // Hooboy, this is fugly. We need a way to communicate to the
2753 // encoder that the index needs to be sign extended, so we have to
2754 // enumerate all the cases.
2755 switch (opcode) {
2756 case INDINDEXSCALEDI2L:
2757 case INDINDEXSCALEDI2LN:
2758 case INDINDEXI2L:
2759 case INDINDEXI2LN:
2760 scale = Address::sxtw(size);
2761 break;
2762 default:
2763 scale = Address::lsl(size);
2764 }
2765
2766 if (index == -1) {
2767 return Address(base, disp);
2768 } else {
2769 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2770 return Address(base, as_Register(index), scale);
2771 }
2772 }
2773
2774
2775 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2776 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2777 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2778 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2779 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2780
2781 // Used for all non-volatile memory accesses. The use of
2782 // $mem->opcode() to discover whether this pattern uses sign-extended
2783 // offsets is something of a kludge.
2784 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2785 Register reg, int opcode,
2786 Register base, int index, int scale, int disp,
2787 int size_in_memory)
2788 {
2789 Address addr = mem2address(opcode, base, index, scale, disp);
2790 if (addr.getMode() == Address::base_plus_offset) {
2791 /* Fix up any out-of-range offsets. */
2792 assert_different_registers(rscratch1, base);
2793 assert_different_registers(rscratch1, reg);
2794 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2795 }
2796 (masm->*insn)(reg, addr);
2797 }
2798
2799 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2800 FloatRegister reg, int opcode,
2801 Register base, int index, int size, int disp,
2802 int size_in_memory)
2803 {
2804 Address::extend scale;
2805
2806 switch (opcode) {
2807 case INDINDEXSCALEDI2L:
2808 case INDINDEXSCALEDI2LN:
2809 scale = Address::sxtw(size);
2810 break;
2811 default:
2812 scale = Address::lsl(size);
2813 }
2814
2815 if (index == -1) {
2816 // Fix up any out-of-range offsets.
2817 assert_different_registers(rscratch1, base);
2818 Address addr = Address(base, disp);
2819 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2820 (masm->*insn)(reg, addr);
2821 } else {
2822 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2823 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2824 }
2825 }
2826
2827 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2828 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2829 int opcode, Register base, int index, int size, int disp)
2830 {
2831 if (index == -1) {
2832 (masm->*insn)(reg, T, Address(base, disp));
2833 } else {
2834 assert(disp == 0, "unsupported address mode");
2835 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2836 }
2837 }
2838
2839 %}
2840
2841
2842
2843 //----------ENCODING BLOCK-----------------------------------------------------
2844 // This block specifies the encoding classes used by the compiler to
2845 // output byte streams. Encoding classes are parameterized macros
2846 // used by Machine Instruction Nodes in order to generate the bit
2847 // encoding of the instruction. Operands specify their base encoding
2848 // interface with the interface keyword. There are currently
2849 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2850 // COND_INTER. REG_INTER causes an operand to generate a function
2851 // which returns its register number when queried. CONST_INTER causes
2852 // an operand to generate a function which returns the value of the
2853 // constant when queried. MEMORY_INTER causes an operand to generate
2854 // four functions which return the Base Register, the Index Register,
2855 // the Scale Value, and the Offset Value of the operand when queried.
2856 // COND_INTER causes an operand to generate six functions which return
2857 // the encoding code (ie - encoding bits for the instruction)
2858 // associated with each basic boolean condition for a conditional
2859 // instruction.
2860 //
2861 // Instructions specify two basic values for encoding. Again, a
2862 // function is available to check if the constant displacement is an
2863 // oop. They use the ins_encode keyword to specify their encoding
2864 // classes (which must be a sequence of enc_class names, and their
2865 // parameters, specified in the encoding block), and they use the
2866 // opcode keyword to specify, in order, their primary, secondary, and
2867 // tertiary opcode. Only the opcode sections which a particular
2868 // instruction needs for encoding need to be specified.
2869 encode %{
2870 // Build emit functions for each basic byte or larger field in the
2871 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2872 // from C++ code in the enc_class source block. Emit functions will
2873 // live in the main source block for now. In future, we can
2874 // generalize this by adding a syntax that specifies the sizes of
2875 // fields in an order, so that the adlc can build the emit functions
2876 // automagically
2877
2878 // catch all for unimplemented encodings
2879 enc_class enc_unimplemented %{
2880 __ unimplemented("C2 catch all");
2881 %}
2882
2883 // BEGIN Non-volatile memory access
2884
2885 // This encoding class is generated automatically from ad_encode.m4.
2886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2887 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2888 Register dst_reg = as_Register($dst$$reg);
2889 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2890 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2891 %}
2892
2893 // This encoding class is generated automatically from ad_encode.m4.
2894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2895 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2896 Register dst_reg = as_Register($dst$$reg);
2897 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2899 %}
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2904 Register dst_reg = as_Register($dst$$reg);
2905 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2912 Register dst_reg = as_Register($dst$$reg);
2913 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2920 Register dst_reg = as_Register($dst$$reg);
2921 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2928 Register dst_reg = as_Register($dst$$reg);
2929 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2936 Register dst_reg = as_Register($dst$$reg);
2937 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2944 Register dst_reg = as_Register($dst$$reg);
2945 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2952 Register dst_reg = as_Register($dst$$reg);
2953 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2960 Register dst_reg = as_Register($dst$$reg);
2961 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2968 Register dst_reg = as_Register($dst$$reg);
2969 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2976 Register dst_reg = as_Register($dst$$reg);
2977 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2978 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2979 %}
2980
2981 // This encoding class is generated automatically from ad_encode.m4.
2982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2983 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2984 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2985 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2986 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2987 %}
2988
2989 // This encoding class is generated automatically from ad_encode.m4.
2990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2991 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2992 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2993 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3000 Register src_reg = as_Register($src$$reg);
3001 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3002 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3003 %}
3004
3005 // This encoding class is generated automatically from ad_encode.m4.
3006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3007 enc_class aarch64_enc_strb0(memory1 mem) %{
3008 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3009 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3010 %}
3011
3012 // This encoding class is generated automatically from ad_encode.m4.
3013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3014 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3015 Register src_reg = as_Register($src$$reg);
3016 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3017 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3018 %}
3019
3020 // This encoding class is generated automatically from ad_encode.m4.
3021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3022 enc_class aarch64_enc_strh0(memory2 mem) %{
3023 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3024 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3025 %}
3026
3027 // This encoding class is generated automatically from ad_encode.m4.
3028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3029 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3030 Register src_reg = as_Register($src$$reg);
3031 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3032 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3033 %}
3034
3035 // This encoding class is generated automatically from ad_encode.m4.
3036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3037 enc_class aarch64_enc_strw0(memory4 mem) %{
3038 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3039 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3040 %}
3041
3042 // This encoding class is generated automatically from ad_encode.m4.
3043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3044 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3045 Register src_reg = as_Register($src$$reg);
3046 // we sometimes get asked to store the stack pointer into the
3047 // current thread -- we cannot do that directly on AArch64
3048 if (src_reg == r31_sp) {
3049 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3050 __ mov(rscratch2, sp);
3051 src_reg = rscratch2;
3052 }
3053 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3054 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3055 %}
3056
3057 // This encoding class is generated automatically from ad_encode.m4.
3058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3059 enc_class aarch64_enc_str0(memory8 mem) %{
3060 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3061 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3062 %}
3063
3064 // This encoding class is generated automatically from ad_encode.m4.
3065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3066 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3067 FloatRegister src_reg = as_FloatRegister($src$$reg);
3068 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3069 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3070 %}
3071
3072 // This encoding class is generated automatically from ad_encode.m4.
3073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3074 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3075 FloatRegister src_reg = as_FloatRegister($src$$reg);
3076 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3077 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3078 %}
3079
3080 // This encoding class is generated automatically from ad_encode.m4.
3081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3082 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3083 __ membar(Assembler::StoreStore);
3084 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3085 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3086 %}
3087
3088 // END Non-volatile memory access
3089
3090 // Vector loads and stores
3091 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3092 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3093 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3094 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3095 %}
3096
3097 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3098 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3099 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3100 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3101 %}
3102
3103 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3104 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3105 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3106 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3107 %}
3108
3109 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3110 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3111 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3112 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3113 %}
3114
3115 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3116 FloatRegister src_reg = as_FloatRegister($src$$reg);
3117 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3118 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3119 %}
3120
3121 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3122 FloatRegister src_reg = as_FloatRegister($src$$reg);
3123 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3124 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3125 %}
3126
3127 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3128 FloatRegister src_reg = as_FloatRegister($src$$reg);
3129 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3130 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3131 %}
3132
3133 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3134 FloatRegister src_reg = as_FloatRegister($src$$reg);
3135 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3136 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3137 %}
3138
3139 // volatile loads and stores
3140
3141 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3142 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3143 rscratch1, stlrb);
3144 %}
3145
3146 enc_class aarch64_enc_stlrb0(memory mem) %{
3147 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3148 rscratch1, stlrb);
3149 %}
3150
3151 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3152 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, stlrh);
3154 %}
3155
3156 enc_class aarch64_enc_stlrh0(memory mem) %{
3157 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, stlrh);
3159 %}
3160
3161 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3162 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3163 rscratch1, stlrw);
3164 %}
3165
3166 enc_class aarch64_enc_stlrw0(memory mem) %{
3167 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3168 rscratch1, stlrw);
3169 %}
3170
3171 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3172 Register dst_reg = as_Register($dst$$reg);
3173 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3174 rscratch1, ldarb);
3175 __ sxtbw(dst_reg, dst_reg);
3176 %}
3177
3178 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3179 Register dst_reg = as_Register($dst$$reg);
3180 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldarb);
3182 __ sxtb(dst_reg, dst_reg);
3183 %}
3184
3185 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3186 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3187 rscratch1, ldarb);
3188 %}
3189
3190 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3191 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3192 rscratch1, ldarb);
3193 %}
3194
3195 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3196 Register dst_reg = as_Register($dst$$reg);
3197 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, ldarh);
3199 __ sxthw(dst_reg, dst_reg);
3200 %}
3201
3202 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3203 Register dst_reg = as_Register($dst$$reg);
3204 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3205 rscratch1, ldarh);
3206 __ sxth(dst_reg, dst_reg);
3207 %}
3208
3209 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3210 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3211 rscratch1, ldarh);
3212 %}
3213
3214 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3215 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3216 rscratch1, ldarh);
3217 %}
3218
3219 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3220 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, ldarw);
3222 %}
3223
3224 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3225 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldarw);
3227 %}
3228
3229 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3230 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3231 rscratch1, ldar);
3232 %}
3233
3234 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3235 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3236 rscratch1, ldarw);
3237 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3238 %}
3239
3240 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3241 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3242 rscratch1, ldar);
3243 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3244 %}
3245
3246 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3247 Register src_reg = as_Register($src$$reg);
3248 // we sometimes get asked to store the stack pointer into the
3249 // current thread -- we cannot do that directly on AArch64
3250 if (src_reg == r31_sp) {
3251 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3252 __ mov(rscratch2, sp);
3253 src_reg = rscratch2;
3254 }
3255 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3256 rscratch1, stlr);
3257 %}
3258
3259 enc_class aarch64_enc_stlr0(memory mem) %{
3260 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3261 rscratch1, stlr);
3262 %}
3263
3264 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3265 {
3266 FloatRegister src_reg = as_FloatRegister($src$$reg);
3267 __ fmovs(rscratch2, src_reg);
3268 }
3269 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3270 rscratch1, stlrw);
3271 %}
3272
3273 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3274 {
3275 FloatRegister src_reg = as_FloatRegister($src$$reg);
3276 __ fmovd(rscratch2, src_reg);
3277 }
3278 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3279 rscratch1, stlr);
3280 %}
3281
3282 // synchronized read/update encodings
3283
3284 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3285 Register dst_reg = as_Register($dst$$reg);
3286 Register base = as_Register($mem$$base);
3287 int index = $mem$$index;
3288 int scale = $mem$$scale;
3289 int disp = $mem$$disp;
3290 if (index == -1) {
3291 if (disp != 0) {
3292 __ lea(rscratch1, Address(base, disp));
3293 __ ldaxr(dst_reg, rscratch1);
3294 } else {
3295 // TODO
3296 // should we ever get anything other than this case?
3297 __ ldaxr(dst_reg, base);
3298 }
3299 } else {
3300 Register index_reg = as_Register(index);
3301 if (disp == 0) {
3302 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3303 __ ldaxr(dst_reg, rscratch1);
3304 } else {
3305 __ lea(rscratch1, Address(base, disp));
3306 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3307 __ ldaxr(dst_reg, rscratch1);
3308 }
3309 }
3310 %}
3311
3312 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3313 Register src_reg = as_Register($src$$reg);
3314 Register base = as_Register($mem$$base);
3315 int index = $mem$$index;
3316 int scale = $mem$$scale;
3317 int disp = $mem$$disp;
3318 if (index == -1) {
3319 if (disp != 0) {
3320 __ lea(rscratch2, Address(base, disp));
3321 __ stlxr(rscratch1, src_reg, rscratch2);
3322 } else {
3323 // TODO
3324 // should we ever get anything other than this case?
3325 __ stlxr(rscratch1, src_reg, base);
3326 }
3327 } else {
3328 Register index_reg = as_Register(index);
3329 if (disp == 0) {
3330 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3331 __ stlxr(rscratch1, src_reg, rscratch2);
3332 } else {
3333 __ lea(rscratch2, Address(base, disp));
3334 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3335 __ stlxr(rscratch1, src_reg, rscratch2);
3336 }
3337 }
3338 __ cmpw(rscratch1, zr);
3339 %}
3340
3341 // prefetch encodings
3342
3343 enc_class aarch64_enc_prefetchw(memory mem) %{
3344 Register base = as_Register($mem$$base);
3345 int index = $mem$$index;
3346 int scale = $mem$$scale;
3347 int disp = $mem$$disp;
3348 if (index == -1) {
3349 // Fix up any out-of-range offsets.
3350 assert_different_registers(rscratch1, base);
3351 Address addr = Address(base, disp);
3352 addr = __ legitimize_address(addr, 8, rscratch1);
3353 __ prfm(addr, PSTL1KEEP);
3354 } else {
3355 Register index_reg = as_Register(index);
3356 if (disp == 0) {
3357 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3358 } else {
3359 __ lea(rscratch1, Address(base, disp));
3360 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3361 }
3362 }
3363 %}
3364
3365 // mov encodings
3366
3367 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3368 uint32_t con = (uint32_t)$src$$constant;
3369 Register dst_reg = as_Register($dst$$reg);
3370 if (con == 0) {
3371 __ movw(dst_reg, zr);
3372 } else {
3373 __ movw(dst_reg, con);
3374 }
3375 %}
3376
3377 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3378 Register dst_reg = as_Register($dst$$reg);
3379 uint64_t con = (uint64_t)$src$$constant;
3380 if (con == 0) {
3381 __ mov(dst_reg, zr);
3382 } else {
3383 __ mov(dst_reg, con);
3384 }
3385 %}
3386
3387 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3388 Register dst_reg = as_Register($dst$$reg);
3389 address con = (address)$src$$constant;
3390 if (con == nullptr || con == (address)1) {
3391 ShouldNotReachHere();
3392 } else {
3393 relocInfo::relocType rtype = $src->constant_reloc();
3394 if (rtype == relocInfo::oop_type) {
3395 __ movoop(dst_reg, (jobject)con);
3396 } else if (rtype == relocInfo::metadata_type) {
3397 __ mov_metadata(dst_reg, (Metadata*)con);
3398 } else {
3399 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3400 // load fake address constants using a normal move
3401 if (! __ is_valid_AArch64_address(con) ||
3402 con < (address)(uintptr_t)os::vm_page_size()) {
3403 __ mov(dst_reg, con);
3404 } else {
3405 // no reloc so just use adrp and add
3406 uint64_t offset;
3407 __ adrp(dst_reg, con, offset);
3408 __ add(dst_reg, dst_reg, offset);
3409 }
3410 }
3411 }
3412 %}
3413
3414 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3415 Register dst_reg = as_Register($dst$$reg);
3416 __ mov(dst_reg, zr);
3417 %}
3418
3419 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3420 Register dst_reg = as_Register($dst$$reg);
3421 __ mov(dst_reg, (uint64_t)1);
3422 %}
3423
3424 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3425 Register dst_reg = as_Register($dst$$reg);
3426 address con = (address)$src$$constant;
3427 if (con == nullptr) {
3428 ShouldNotReachHere();
3429 } else {
3430 relocInfo::relocType rtype = $src->constant_reloc();
3431 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3432 __ set_narrow_oop(dst_reg, (jobject)con);
3433 }
3434 %}
3435
3436 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3437 Register dst_reg = as_Register($dst$$reg);
3438 __ mov(dst_reg, zr);
3439 %}
3440
3441 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3442 Register dst_reg = as_Register($dst$$reg);
3443 address con = (address)$src$$constant;
3444 if (con == nullptr) {
3445 ShouldNotReachHere();
3446 } else {
3447 relocInfo::relocType rtype = $src->constant_reloc();
3448 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3449 __ set_narrow_klass(dst_reg, (Klass *)con);
3450 }
3451 %}
3452
3453 // arithmetic encodings
3454
3455 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3456 Register dst_reg = as_Register($dst$$reg);
3457 Register src_reg = as_Register($src1$$reg);
3458 int32_t con = (int32_t)$src2$$constant;
3459 // add has primary == 0, subtract has primary == 1
3460 if ($primary) { con = -con; }
3461 if (con < 0) {
3462 __ subw(dst_reg, src_reg, -con);
3463 } else {
3464 __ addw(dst_reg, src_reg, con);
3465 }
3466 %}
3467
3468 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3469 Register dst_reg = as_Register($dst$$reg);
3470 Register src_reg = as_Register($src1$$reg);
3471 int32_t con = (int32_t)$src2$$constant;
3472 // add has primary == 0, subtract has primary == 1
3473 if ($primary) { con = -con; }
3474 if (con < 0) {
3475 __ sub(dst_reg, src_reg, -con);
3476 } else {
3477 __ add(dst_reg, src_reg, con);
3478 }
3479 %}
3480
3481 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3482 Register dst_reg = as_Register($dst$$reg);
3483 Register src1_reg = as_Register($src1$$reg);
3484 Register src2_reg = as_Register($src2$$reg);
3485 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3486 %}
3487
3488 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3489 Register dst_reg = as_Register($dst$$reg);
3490 Register src1_reg = as_Register($src1$$reg);
3491 Register src2_reg = as_Register($src2$$reg);
3492 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3493 %}
3494
3495 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3496 Register dst_reg = as_Register($dst$$reg);
3497 Register src1_reg = as_Register($src1$$reg);
3498 Register src2_reg = as_Register($src2$$reg);
3499 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3500 %}
3501
3502 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3503 Register dst_reg = as_Register($dst$$reg);
3504 Register src1_reg = as_Register($src1$$reg);
3505 Register src2_reg = as_Register($src2$$reg);
3506 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3507 %}
3508
3509 // compare instruction encodings
3510
3511 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3512 Register reg1 = as_Register($src1$$reg);
3513 Register reg2 = as_Register($src2$$reg);
3514 __ cmpw(reg1, reg2);
3515 %}
3516
3517 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3518 Register reg = as_Register($src1$$reg);
3519 int32_t val = $src2$$constant;
3520 if (val >= 0) {
3521 __ subsw(zr, reg, val);
3522 } else {
3523 __ addsw(zr, reg, -val);
3524 }
3525 %}
3526
3527 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3528 Register reg1 = as_Register($src1$$reg);
3529 uint32_t val = (uint32_t)$src2$$constant;
3530 __ movw(rscratch1, val);
3531 __ cmpw(reg1, rscratch1);
3532 %}
3533
3534 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3535 Register reg1 = as_Register($src1$$reg);
3536 Register reg2 = as_Register($src2$$reg);
3537 __ cmp(reg1, reg2);
3538 %}
3539
3540 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3541 Register reg = as_Register($src1$$reg);
3542 int64_t val = $src2$$constant;
3543 if (val >= 0) {
3544 __ subs(zr, reg, val);
3545 } else if (val != -val) {
3546 __ adds(zr, reg, -val);
3547 } else {
3548 // aargh, Long.MIN_VALUE is a special case
3549 __ orr(rscratch1, zr, (uint64_t)val);
3550 __ subs(zr, reg, rscratch1);
3551 }
3552 %}
3553
3554 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3555 Register reg1 = as_Register($src1$$reg);
3556 uint64_t val = (uint64_t)$src2$$constant;
3557 __ mov(rscratch1, val);
3558 __ cmp(reg1, rscratch1);
3559 %}
3560
3561 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3562 Register reg1 = as_Register($src1$$reg);
3563 Register reg2 = as_Register($src2$$reg);
3564 __ cmp(reg1, reg2);
3565 %}
3566
3567 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3568 Register reg1 = as_Register($src1$$reg);
3569 Register reg2 = as_Register($src2$$reg);
3570 __ cmpw(reg1, reg2);
3571 %}
3572
3573 enc_class aarch64_enc_testp(iRegP src) %{
3574 Register reg = as_Register($src$$reg);
3575 __ cmp(reg, zr);
3576 %}
3577
3578 enc_class aarch64_enc_testn(iRegN src) %{
3579 Register reg = as_Register($src$$reg);
3580 __ cmpw(reg, zr);
3581 %}
3582
3583 enc_class aarch64_enc_b(label lbl) %{
3584 Label *L = $lbl$$label;
3585 __ b(*L);
3586 %}
3587
3588 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3589 Label *L = $lbl$$label;
3590 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3591 %}
3592
3593 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3594 Label *L = $lbl$$label;
3595 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3596 %}
3597
3598 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3599 %{
3600 Register sub_reg = as_Register($sub$$reg);
3601 Register super_reg = as_Register($super$$reg);
3602 Register temp_reg = as_Register($temp$$reg);
3603 Register result_reg = as_Register($result$$reg);
3604
3605 Label miss;
3606 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3607 nullptr, &miss,
3608 /*set_cond_codes:*/ true);
3609 if ($primary) {
3610 __ mov(result_reg, zr);
3611 }
3612 __ bind(miss);
3613 %}
3614
3615 enc_class aarch64_enc_java_static_call(method meth) %{
3616 address addr = (address)$meth$$method;
3617 address call;
3618 if (!_method) {
3619 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3620 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3621 if (call == nullptr) {
3622 ciEnv::current()->record_failure("CodeCache is full");
3623 return;
3624 }
3625 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3626 // The NOP here is purely to ensure that eliding a call to
3627 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3628 __ nop();
3629 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3630 } else {
3631 int method_index = resolved_method_index(masm);
3632 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3633 : static_call_Relocation::spec(method_index);
3634 call = __ trampoline_call(Address(addr, rspec));
3635 if (call == nullptr) {
3636 ciEnv::current()->record_failure("CodeCache is full");
3637 return;
3638 }
3639 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3640 // Calls of the same statically bound method can share
3641 // a stub to the interpreter.
3642 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3643 } else {
3644 // Emit stub for static call
3645 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3646 if (stub == nullptr) {
3647 ciEnv::current()->record_failure("CodeCache is full");
3648 return;
3649 }
3650 }
3651 }
3652
3653 __ post_call_nop();
3654
3655 // Only non uncommon_trap calls need to reinitialize ptrue.
3656 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3657 __ reinitialize_ptrue();
3658 }
3659 %}
3660
3661 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3662 int method_index = resolved_method_index(masm);
3663 address call = __ ic_call((address)$meth$$method, method_index);
3664 if (call == nullptr) {
3665 ciEnv::current()->record_failure("CodeCache is full");
3666 return;
3667 }
3668 __ post_call_nop();
3669 if (Compile::current()->max_vector_size() > 0) {
3670 __ reinitialize_ptrue();
3671 }
3672 %}
3673
3674 enc_class aarch64_enc_call_epilog() %{
3675 if (VerifyStackAtCalls) {
3676 // Check that stack depth is unchanged: find majik cookie on stack
3677 __ call_Unimplemented();
3678 }
3679 %}
3680
3681 enc_class aarch64_enc_java_to_runtime(method meth) %{
3682 // some calls to generated routines (arraycopy code) are scheduled
3683 // by C2 as runtime calls. if so we can call them using a br (they
3684 // will be in a reachable segment) otherwise we have to use a blr
3685 // which loads the absolute address into a register.
3686 address entry = (address)$meth$$method;
3687 CodeBlob *cb = CodeCache::find_blob(entry);
3688 if (cb) {
3689 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3690 if (call == nullptr) {
3691 ciEnv::current()->record_failure("CodeCache is full");
3692 return;
3693 }
3694 __ post_call_nop();
3695 } else {
3696 Label retaddr;
3697 // Make the anchor frame walkable
3698 __ adr(rscratch2, retaddr);
3699 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3700 __ lea(rscratch1, RuntimeAddress(entry));
3701 __ blr(rscratch1);
3702 __ bind(retaddr);
3703 __ post_call_nop();
3704 }
3705 if (Compile::current()->max_vector_size() > 0) {
3706 __ reinitialize_ptrue();
3707 }
3708 %}
3709
3710 enc_class aarch64_enc_rethrow() %{
3711 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3712 %}
3713
3714 enc_class aarch64_enc_ret() %{
3715 #ifdef ASSERT
3716 if (Compile::current()->max_vector_size() > 0) {
3717 __ verify_ptrue();
3718 }
3719 #endif
3720 __ ret(lr);
3721 %}
3722
3723 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3724 Register target_reg = as_Register($jump_target$$reg);
3725 __ br(target_reg);
3726 %}
3727
3728 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3729 Register target_reg = as_Register($jump_target$$reg);
3730 // exception oop should be in r0
3731 // ret addr has been popped into lr
3732 // callee expects it in r3
3733 __ mov(r3, lr);
3734 __ br(target_reg);
3735 %}
3736
3737 %}
3738
3739 //----------FRAME--------------------------------------------------------------
3740 // Definition of frame structure and management information.
3741 //
3742 // S T A C K L A Y O U T Allocators stack-slot number
3743 // | (to get allocators register number
3744 // G Owned by | | v add OptoReg::stack0())
3745 // r CALLER | |
3746 // o | +--------+ pad to even-align allocators stack-slot
3747 // w V | pad0 | numbers; owned by CALLER
3748 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3749 // h ^ | in | 5
3750 // | | args | 4 Holes in incoming args owned by SELF
3751 // | | | | 3
3752 // | | +--------+
3753 // V | | old out| Empty on Intel, window on Sparc
3754 // | old |preserve| Must be even aligned.
3755 // | SP-+--------+----> Matcher::_old_SP, even aligned
3756 // | | in | 3 area for Intel ret address
3757 // Owned by |preserve| Empty on Sparc.
3758 // SELF +--------+
3759 // | | pad2 | 2 pad to align old SP
3760 // | +--------+ 1
3761 // | | locks | 0
3762 // | +--------+----> OptoReg::stack0(), even aligned
3763 // | | pad1 | 11 pad to align new SP
3764 // | +--------+
3765 // | | | 10
3766 // | | spills | 9 spills
3767 // V | | 8 (pad0 slot for callee)
3768 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3769 // ^ | out | 7
3770 // | | args | 6 Holes in outgoing args owned by CALLEE
3771 // Owned by +--------+
3772 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3773 // | new |preserve| Must be even-aligned.
3774 // | SP-+--------+----> Matcher::_new_SP, even aligned
3775 // | | |
3776 //
3777 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3778 // known from SELF's arguments and the Java calling convention.
3779 // Region 6-7 is determined per call site.
3780 // Note 2: If the calling convention leaves holes in the incoming argument
3781 // area, those holes are owned by SELF. Holes in the outgoing area
3782 // are owned by the CALLEE. Holes should not be necessary in the
3783 // incoming area, as the Java calling convention is completely under
3784 // the control of the AD file. Doubles can be sorted and packed to
3785 // avoid holes. Holes in the outgoing arguments may be necessary for
3786 // varargs C calling conventions.
3787 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3788 // even aligned with pad0 as needed.
3789 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3790 // (the latter is true on Intel but is it false on AArch64?)
3791 // region 6-11 is even aligned; it may be padded out more so that
3792 // the region from SP to FP meets the minimum stack alignment.
3793 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3794 // alignment. Region 11, pad1, may be dynamically extended so that
3795 // SP meets the minimum alignment.
3796
3797 frame %{
3798 // These three registers define part of the calling convention
3799 // between compiled code and the interpreter.
3800
3801 // Inline Cache Register or Method for I2C.
3802 inline_cache_reg(R12);
3803
3804 // Number of stack slots consumed by locking an object
3805 sync_stack_slots(2);
3806
3807 // Compiled code's Frame Pointer
3808 frame_pointer(R31);
3809
3810 // Stack alignment requirement
3811 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3812
3813 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3814 // for calls to C. Supports the var-args backing area for register parms.
3815 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3816
3817 // The after-PROLOG location of the return address. Location of
3818 // return address specifies a type (REG or STACK) and a number
3819 // representing the register number (i.e. - use a register name) or
3820 // stack slot.
3821 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3822 // Otherwise, it is above the locks and verification slot and alignment word
3823 // TODO this may well be correct but need to check why that - 2 is there
3824 // ppc port uses 0 but we definitely need to allow for fixed_slots
3825 // which folds in the space used for monitors
3826 return_addr(STACK - 2 +
3827 align_up((Compile::current()->in_preserve_stack_slots() +
3828 Compile::current()->fixed_slots()),
3829 stack_alignment_in_slots()));
3830
3831 // Location of compiled Java return values. Same as C for now.
3832 return_value
3833 %{
3834 // TODO do we allow ideal_reg == Op_RegN???
3835 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3836 "only return normal values");
3837
3838 static const int lo[Op_RegL + 1] = { // enum name
3839 0, // Op_Node
3840 0, // Op_Set
3841 R0_num, // Op_RegN
3842 R0_num, // Op_RegI
3843 R0_num, // Op_RegP
3844 V0_num, // Op_RegF
3845 V0_num, // Op_RegD
3846 R0_num // Op_RegL
3847 };
3848
3849 static const int hi[Op_RegL + 1] = { // enum name
3850 0, // Op_Node
3851 0, // Op_Set
3852 OptoReg::Bad, // Op_RegN
3853 OptoReg::Bad, // Op_RegI
3854 R0_H_num, // Op_RegP
3855 OptoReg::Bad, // Op_RegF
3856 V0_H_num, // Op_RegD
3857 R0_H_num // Op_RegL
3858 };
3859
3860 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3861 %}
3862 %}
3863
3864 //----------ATTRIBUTES---------------------------------------------------------
3865 //----------Operand Attributes-------------------------------------------------
3866 op_attrib op_cost(1); // Required cost attribute
3867
3868 //----------Instruction Attributes---------------------------------------------
3869 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3870 ins_attrib ins_size(32); // Required size attribute (in bits)
3871 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3872 // a non-matching short branch variant
3873 // of some long branch?
3874 ins_attrib ins_alignment(4); // Required alignment attribute (must
3875 // be a power of 2) specifies the
3876 // alignment that some part of the
3877 // instruction (not necessarily the
3878 // start) requires. If > 1, a
3879 // compute_padding() function must be
3880 // provided for the instruction
3881
3882 // Whether this node is expanded during code emission into a sequence of
3883 // instructions and the first instruction can perform an implicit null check.
3884 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3885
3886 //----------OPERANDS-----------------------------------------------------------
3887 // Operand definitions must precede instruction definitions for correct parsing
3888 // in the ADLC because operands constitute user defined types which are used in
3889 // instruction definitions.
3890
3891 //----------Simple Operands----------------------------------------------------
3892
3893 // Integer operands 32 bit
3894 // 32 bit immediate
3895 operand immI()
3896 %{
3897 match(ConI);
3898
3899 op_cost(0);
3900 format %{ %}
3901 interface(CONST_INTER);
3902 %}
3903
3904 // 32 bit zero
3905 operand immI0()
3906 %{
3907 predicate(n->get_int() == 0);
3908 match(ConI);
3909
3910 op_cost(0);
3911 format %{ %}
3912 interface(CONST_INTER);
3913 %}
3914
3915 // 32 bit unit increment
3916 operand immI_1()
3917 %{
3918 predicate(n->get_int() == 1);
3919 match(ConI);
3920
3921 op_cost(0);
3922 format %{ %}
3923 interface(CONST_INTER);
3924 %}
3925
3926 // 32 bit unit decrement
3927 operand immI_M1()
3928 %{
3929 predicate(n->get_int() == -1);
3930 match(ConI);
3931
3932 op_cost(0);
3933 format %{ %}
3934 interface(CONST_INTER);
3935 %}
3936
3937 // Shift values for add/sub extension shift
3938 operand immIExt()
3939 %{
3940 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3941 match(ConI);
3942
3943 op_cost(0);
3944 format %{ %}
3945 interface(CONST_INTER);
3946 %}
3947
3948 operand immI_gt_1()
3949 %{
3950 predicate(n->get_int() > 1);
3951 match(ConI);
3952
3953 op_cost(0);
3954 format %{ %}
3955 interface(CONST_INTER);
3956 %}
3957
3958 operand immI_le_4()
3959 %{
3960 predicate(n->get_int() <= 4);
3961 match(ConI);
3962
3963 op_cost(0);
3964 format %{ %}
3965 interface(CONST_INTER);
3966 %}
3967
3968 operand immI_16()
3969 %{
3970 predicate(n->get_int() == 16);
3971 match(ConI);
3972
3973 op_cost(0);
3974 format %{ %}
3975 interface(CONST_INTER);
3976 %}
3977
3978 operand immI_24()
3979 %{
3980 predicate(n->get_int() == 24);
3981 match(ConI);
3982
3983 op_cost(0);
3984 format %{ %}
3985 interface(CONST_INTER);
3986 %}
3987
3988 operand immI_32()
3989 %{
3990 predicate(n->get_int() == 32);
3991 match(ConI);
3992
3993 op_cost(0);
3994 format %{ %}
3995 interface(CONST_INTER);
3996 %}
3997
3998 operand immI_48()
3999 %{
4000 predicate(n->get_int() == 48);
4001 match(ConI);
4002
4003 op_cost(0);
4004 format %{ %}
4005 interface(CONST_INTER);
4006 %}
4007
4008 operand immI_56()
4009 %{
4010 predicate(n->get_int() == 56);
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_255()
4019 %{
4020 predicate(n->get_int() == 255);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_65535()
4029 %{
4030 predicate(n->get_int() == 65535);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 operand immI_positive()
4039 %{
4040 predicate(n->get_int() > 0);
4041 match(ConI);
4042
4043 op_cost(0);
4044 format %{ %}
4045 interface(CONST_INTER);
4046 %}
4047
4048 // BoolTest condition for signed compare
4049 operand immI_cmp_cond()
4050 %{
4051 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4052 match(ConI);
4053
4054 op_cost(0);
4055 format %{ %}
4056 interface(CONST_INTER);
4057 %}
4058
4059 // BoolTest condition for unsigned compare
4060 operand immI_cmpU_cond()
4061 %{
4062 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4063 match(ConI);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 operand immL_255()
4071 %{
4072 predicate(n->get_long() == 255L);
4073 match(ConL);
4074
4075 op_cost(0);
4076 format %{ %}
4077 interface(CONST_INTER);
4078 %}
4079
4080 operand immL_65535()
4081 %{
4082 predicate(n->get_long() == 65535L);
4083 match(ConL);
4084
4085 op_cost(0);
4086 format %{ %}
4087 interface(CONST_INTER);
4088 %}
4089
4090 operand immL_4294967295()
4091 %{
4092 predicate(n->get_long() == 4294967295L);
4093 match(ConL);
4094
4095 op_cost(0);
4096 format %{ %}
4097 interface(CONST_INTER);
4098 %}
4099
4100 operand immL_bitmask()
4101 %{
4102 predicate((n->get_long() != 0)
4103 && ((n->get_long() & 0xc000000000000000l) == 0)
4104 && is_power_of_2(n->get_long() + 1));
4105 match(ConL);
4106
4107 op_cost(0);
4108 format %{ %}
4109 interface(CONST_INTER);
4110 %}
4111
4112 operand immI_bitmask()
4113 %{
4114 predicate((n->get_int() != 0)
4115 && ((n->get_int() & 0xc0000000) == 0)
4116 && is_power_of_2(n->get_int() + 1));
4117 match(ConI);
4118
4119 op_cost(0);
4120 format %{ %}
4121 interface(CONST_INTER);
4122 %}
4123
4124 operand immL_positive_bitmaskI()
4125 %{
4126 predicate((n->get_long() != 0)
4127 && ((julong)n->get_long() < 0x80000000ULL)
4128 && is_power_of_2(n->get_long() + 1));
4129 match(ConL);
4130
4131 op_cost(0);
4132 format %{ %}
4133 interface(CONST_INTER);
4134 %}
4135
4136 // Scale values for scaled offset addressing modes (up to long but not quad)
4137 operand immIScale()
4138 %{
4139 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4140 match(ConI);
4141
4142 op_cost(0);
4143 format %{ %}
4144 interface(CONST_INTER);
4145 %}
4146
4147 // 5 bit signed integer
4148 operand immI5()
4149 %{
4150 predicate(Assembler::is_simm(n->get_int(), 5));
4151 match(ConI);
4152
4153 op_cost(0);
4154 format %{ %}
4155 interface(CONST_INTER);
4156 %}
4157
4158 // 7 bit unsigned integer
4159 operand immIU7()
4160 %{
4161 predicate(Assembler::is_uimm(n->get_int(), 7));
4162 match(ConI);
4163
4164 op_cost(0);
4165 format %{ %}
4166 interface(CONST_INTER);
4167 %}
4168
4169 // Offset for scaled or unscaled immediate loads and stores
4170 operand immIOffset()
4171 %{
4172 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4173 match(ConI);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 operand immIOffset1()
4181 %{
4182 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4183 match(ConI);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 operand immIOffset2()
4191 %{
4192 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4193 match(ConI);
4194
4195 op_cost(0);
4196 format %{ %}
4197 interface(CONST_INTER);
4198 %}
4199
4200 operand immIOffset4()
4201 %{
4202 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 operand immIOffset8()
4211 %{
4212 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immIOffset16()
4221 %{
4222 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4223 match(ConI);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immLOffset()
4231 %{
4232 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4233 match(ConL);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immLoffset1()
4241 %{
4242 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4243 match(ConL);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immLoffset2()
4251 %{
4252 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4253 match(ConL);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immLoffset4()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4263 match(ConL);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immLoffset8()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4273 match(ConL);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immLoffset16()
4281 %{
4282 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4283 match(ConL);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 // 5 bit signed long integer
4291 operand immL5()
4292 %{
4293 predicate(Assembler::is_simm(n->get_long(), 5));
4294 match(ConL);
4295
4296 op_cost(0);
4297 format %{ %}
4298 interface(CONST_INTER);
4299 %}
4300
4301 // 7 bit unsigned long integer
4302 operand immLU7()
4303 %{
4304 predicate(Assembler::is_uimm(n->get_long(), 7));
4305 match(ConL);
4306
4307 op_cost(0);
4308 format %{ %}
4309 interface(CONST_INTER);
4310 %}
4311
4312 // 8 bit signed value.
4313 operand immI8()
4314 %{
4315 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4316 match(ConI);
4317
4318 op_cost(0);
4319 format %{ %}
4320 interface(CONST_INTER);
4321 %}
4322
4323 // 8 bit signed value (simm8), or #simm8 LSL 8.
4324 operand immIDupV()
4325 %{
4326 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4327 match(ConI);
4328
4329 op_cost(0);
4330 format %{ %}
4331 interface(CONST_INTER);
4332 %}
4333
4334 // 8 bit signed value (simm8), or #simm8 LSL 8.
4335 operand immLDupV()
4336 %{
4337 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4338 match(ConL);
4339
4340 op_cost(0);
4341 format %{ %}
4342 interface(CONST_INTER);
4343 %}
4344
4345 // 8 bit signed value (simm8), or #simm8 LSL 8.
4346 operand immHDupV()
4347 %{
4348 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4349 match(ConH);
4350
4351 op_cost(0);
4352 format %{ %}
4353 interface(CONST_INTER);
4354 %}
4355
4356 // 8 bit integer valid for vector add sub immediate
4357 operand immBAddSubV()
4358 %{
4359 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4360 match(ConI);
4361
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 // 32 bit integer valid for add sub immediate
4368 operand immIAddSub()
4369 %{
4370 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4371 match(ConI);
4372 op_cost(0);
4373 format %{ %}
4374 interface(CONST_INTER);
4375 %}
4376
4377 // 32 bit integer valid for vector add sub immediate
4378 operand immIAddSubV()
4379 %{
4380 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4381 match(ConI);
4382
4383 op_cost(0);
4384 format %{ %}
4385 interface(CONST_INTER);
4386 %}
4387
4388 // 32 bit unsigned integer valid for logical immediate
4389
4390 operand immBLog()
4391 %{
4392 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4393 match(ConI);
4394
4395 op_cost(0);
4396 format %{ %}
4397 interface(CONST_INTER);
4398 %}
4399
4400 operand immSLog()
4401 %{
4402 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4403 match(ConI);
4404
4405 op_cost(0);
4406 format %{ %}
4407 interface(CONST_INTER);
4408 %}
4409
4410 operand immILog()
4411 %{
4412 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4413 match(ConI);
4414
4415 op_cost(0);
4416 format %{ %}
4417 interface(CONST_INTER);
4418 %}
4419
4420 // Integer operands 64 bit
4421 // 64 bit immediate
4422 operand immL()
4423 %{
4424 match(ConL);
4425
4426 op_cost(0);
4427 format %{ %}
4428 interface(CONST_INTER);
4429 %}
4430
4431 // 64 bit zero
4432 operand immL0()
4433 %{
4434 predicate(n->get_long() == 0);
4435 match(ConL);
4436
4437 op_cost(0);
4438 format %{ %}
4439 interface(CONST_INTER);
4440 %}
4441
4442 // 64 bit unit decrement
4443 operand immL_M1()
4444 %{
4445 predicate(n->get_long() == -1);
4446 match(ConL);
4447
4448 op_cost(0);
4449 format %{ %}
4450 interface(CONST_INTER);
4451 %}
4452
4453 // 64 bit integer valid for add sub immediate
4454 operand immLAddSub()
4455 %{
4456 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4457 match(ConL);
4458 op_cost(0);
4459 format %{ %}
4460 interface(CONST_INTER);
4461 %}
4462
4463 // 64 bit integer valid for addv subv immediate
4464 operand immLAddSubV()
4465 %{
4466 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4467 match(ConL);
4468
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // 64 bit integer valid for logical immediate
4475 operand immLLog()
4476 %{
4477 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4478 match(ConL);
4479 op_cost(0);
4480 format %{ %}
4481 interface(CONST_INTER);
4482 %}
4483
4484 // Long Immediate: low 32-bit mask
4485 operand immL_32bits()
4486 %{
4487 predicate(n->get_long() == 0xFFFFFFFFL);
4488 match(ConL);
4489 op_cost(0);
4490 format %{ %}
4491 interface(CONST_INTER);
4492 %}
4493
4494 // Pointer operands
4495 // Pointer Immediate
4496 operand immP()
4497 %{
4498 match(ConP);
4499
4500 op_cost(0);
4501 format %{ %}
4502 interface(CONST_INTER);
4503 %}
4504
4505 // nullptr Pointer Immediate
4506 operand immP0()
4507 %{
4508 predicate(n->get_ptr() == 0);
4509 match(ConP);
4510
4511 op_cost(0);
4512 format %{ %}
4513 interface(CONST_INTER);
4514 %}
4515
4516 // Pointer Immediate One
4517 // this is used in object initialization (initial object header)
4518 operand immP_1()
4519 %{
4520 predicate(n->get_ptr() == 1);
4521 match(ConP);
4522
4523 op_cost(0);
4524 format %{ %}
4525 interface(CONST_INTER);
4526 %}
4527
4528 // AOT Runtime Constants Address
4529 operand immAOTRuntimeConstantsAddress()
4530 %{
4531 // Check if the address is in the range of AOT Runtime Constants
4532 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4533 match(ConP);
4534
4535 op_cost(0);
4536 format %{ %}
4537 interface(CONST_INTER);
4538 %}
4539
4540 // Float and Double operands
4541 // Double Immediate
4542 operand immD()
4543 %{
4544 match(ConD);
4545 op_cost(0);
4546 format %{ %}
4547 interface(CONST_INTER);
4548 %}
4549
4550 // Double Immediate: +0.0d
4551 operand immD0()
4552 %{
4553 predicate(jlong_cast(n->getd()) == 0);
4554 match(ConD);
4555
4556 op_cost(0);
4557 format %{ %}
4558 interface(CONST_INTER);
4559 %}
4560
4561 // constant 'double +0.0'.
4562 operand immDPacked()
4563 %{
4564 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4565 match(ConD);
4566 op_cost(0);
4567 format %{ %}
4568 interface(CONST_INTER);
4569 %}
4570
4571 // Float Immediate
4572 operand immF()
4573 %{
4574 match(ConF);
4575 op_cost(0);
4576 format %{ %}
4577 interface(CONST_INTER);
4578 %}
4579
4580 // Float Immediate: +0.0f.
4581 operand immF0()
4582 %{
4583 predicate(jint_cast(n->getf()) == 0);
4584 match(ConF);
4585
4586 op_cost(0);
4587 format %{ %}
4588 interface(CONST_INTER);
4589 %}
4590
4591 // Half Float (FP16) Immediate
4592 operand immH()
4593 %{
4594 match(ConH);
4595 op_cost(0);
4596 format %{ %}
4597 interface(CONST_INTER);
4598 %}
4599
4600 //
4601 operand immFPacked()
4602 %{
4603 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4604 match(ConF);
4605 op_cost(0);
4606 format %{ %}
4607 interface(CONST_INTER);
4608 %}
4609
4610 // Narrow pointer operands
4611 // Narrow Pointer Immediate
4612 operand immN()
4613 %{
4614 match(ConN);
4615
4616 op_cost(0);
4617 format %{ %}
4618 interface(CONST_INTER);
4619 %}
4620
4621 // Narrow nullptr Pointer Immediate
4622 operand immN0()
4623 %{
4624 predicate(n->get_narrowcon() == 0);
4625 match(ConN);
4626
4627 op_cost(0);
4628 format %{ %}
4629 interface(CONST_INTER);
4630 %}
4631
4632 operand immNKlass()
4633 %{
4634 match(ConNKlass);
4635
4636 op_cost(0);
4637 format %{ %}
4638 interface(CONST_INTER);
4639 %}
4640
4641 // Integer 32 bit Register Operands
4642 // Integer 32 bitRegister (excludes SP)
4643 operand iRegI()
4644 %{
4645 constraint(ALLOC_IN_RC(any_reg32));
4646 match(RegI);
4647 match(iRegINoSp);
4648 op_cost(0);
4649 format %{ %}
4650 interface(REG_INTER);
4651 %}
4652
4653 // Integer 32 bit Register not Special
4654 operand iRegINoSp()
4655 %{
4656 constraint(ALLOC_IN_RC(no_special_reg32));
4657 match(RegI);
4658 op_cost(0);
4659 format %{ %}
4660 interface(REG_INTER);
4661 %}
4662
4663 // Integer 64 bit Register Operands
4664 // Integer 64 bit Register (includes SP)
4665 operand iRegL()
4666 %{
4667 constraint(ALLOC_IN_RC(any_reg));
4668 match(RegL);
4669 match(iRegLNoSp);
4670 op_cost(0);
4671 format %{ %}
4672 interface(REG_INTER);
4673 %}
4674
4675 // Integer 64 bit Register not Special
4676 operand iRegLNoSp()
4677 %{
4678 constraint(ALLOC_IN_RC(no_special_reg));
4679 match(RegL);
4680 match(iRegL_R0);
4681 format %{ %}
4682 interface(REG_INTER);
4683 %}
4684
4685 // Pointer Register Operands
4686 // Pointer Register
4687 operand iRegP()
4688 %{
4689 constraint(ALLOC_IN_RC(ptr_reg));
4690 match(RegP);
4691 match(iRegPNoSp);
4692 match(iRegP_R0);
4693 //match(iRegP_R2);
4694 //match(iRegP_R4);
4695 match(iRegP_R5);
4696 match(thread_RegP);
4697 op_cost(0);
4698 format %{ %}
4699 interface(REG_INTER);
4700 %}
4701
4702 // Pointer 64 bit Register not Special
4703 operand iRegPNoSp()
4704 %{
4705 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4706 match(RegP);
4707 // match(iRegP);
4708 // match(iRegP_R0);
4709 // match(iRegP_R2);
4710 // match(iRegP_R4);
4711 // match(iRegP_R5);
4712 // match(thread_RegP);
4713 op_cost(0);
4714 format %{ %}
4715 interface(REG_INTER);
4716 %}
4717
4718 // This operand is not allowed to use rfp even if
4719 // rfp is not used to hold the frame pointer.
4720 operand iRegPNoSpNoRfp()
4721 %{
4722 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4723 match(RegP);
4724 match(iRegPNoSp);
4725 op_cost(0);
4726 format %{ %}
4727 interface(REG_INTER);
4728 %}
4729
4730 // Pointer 64 bit Register R0 only
4731 operand iRegP_R0()
4732 %{
4733 constraint(ALLOC_IN_RC(r0_reg));
4734 match(RegP);
4735 // match(iRegP);
4736 match(iRegPNoSp);
4737 op_cost(0);
4738 format %{ %}
4739 interface(REG_INTER);
4740 %}
4741
4742 // Pointer 64 bit Register R1 only
4743 operand iRegP_R1()
4744 %{
4745 constraint(ALLOC_IN_RC(r1_reg));
4746 match(RegP);
4747 // match(iRegP);
4748 match(iRegPNoSp);
4749 op_cost(0);
4750 format %{ %}
4751 interface(REG_INTER);
4752 %}
4753
4754 // Pointer 64 bit Register R2 only
4755 operand iRegP_R2()
4756 %{
4757 constraint(ALLOC_IN_RC(r2_reg));
4758 match(RegP);
4759 // match(iRegP);
4760 match(iRegPNoSp);
4761 op_cost(0);
4762 format %{ %}
4763 interface(REG_INTER);
4764 %}
4765
4766 // Pointer 64 bit Register R3 only
4767 operand iRegP_R3()
4768 %{
4769 constraint(ALLOC_IN_RC(r3_reg));
4770 match(RegP);
4771 // match(iRegP);
4772 match(iRegPNoSp);
4773 op_cost(0);
4774 format %{ %}
4775 interface(REG_INTER);
4776 %}
4777
4778 // Pointer 64 bit Register R4 only
4779 operand iRegP_R4()
4780 %{
4781 constraint(ALLOC_IN_RC(r4_reg));
4782 match(RegP);
4783 // match(iRegP);
4784 match(iRegPNoSp);
4785 op_cost(0);
4786 format %{ %}
4787 interface(REG_INTER);
4788 %}
4789
4790 // Pointer 64 bit Register R5 only
4791 operand iRegP_R5()
4792 %{
4793 constraint(ALLOC_IN_RC(r5_reg));
4794 match(RegP);
4795 // match(iRegP);
4796 match(iRegPNoSp);
4797 op_cost(0);
4798 format %{ %}
4799 interface(REG_INTER);
4800 %}
4801
4802 // Pointer 64 bit Register R10 only
4803 operand iRegP_R10()
4804 %{
4805 constraint(ALLOC_IN_RC(r10_reg));
4806 match(RegP);
4807 // match(iRegP);
4808 match(iRegPNoSp);
4809 op_cost(0);
4810 format %{ %}
4811 interface(REG_INTER);
4812 %}
4813
4814 // Long 64 bit Register R0 only
4815 operand iRegL_R0()
4816 %{
4817 constraint(ALLOC_IN_RC(r0_reg));
4818 match(RegL);
4819 match(iRegLNoSp);
4820 op_cost(0);
4821 format %{ %}
4822 interface(REG_INTER);
4823 %}
4824
4825 // Long 64 bit Register R11 only
4826 operand iRegL_R11()
4827 %{
4828 constraint(ALLOC_IN_RC(r11_reg));
4829 match(RegL);
4830 match(iRegLNoSp);
4831 op_cost(0);
4832 format %{ %}
4833 interface(REG_INTER);
4834 %}
4835
4836 // Register R0 only
4837 operand iRegI_R0()
4838 %{
4839 constraint(ALLOC_IN_RC(int_r0_reg));
4840 match(RegI);
4841 match(iRegINoSp);
4842 op_cost(0);
4843 format %{ %}
4844 interface(REG_INTER);
4845 %}
4846
4847 // Register R2 only
4848 operand iRegI_R2()
4849 %{
4850 constraint(ALLOC_IN_RC(int_r2_reg));
4851 match(RegI);
4852 match(iRegINoSp);
4853 op_cost(0);
4854 format %{ %}
4855 interface(REG_INTER);
4856 %}
4857
4858 // Register R3 only
4859 operand iRegI_R3()
4860 %{
4861 constraint(ALLOC_IN_RC(int_r3_reg));
4862 match(RegI);
4863 match(iRegINoSp);
4864 op_cost(0);
4865 format %{ %}
4866 interface(REG_INTER);
4867 %}
4868
4869
4870 // Register R4 only
4871 operand iRegI_R4()
4872 %{
4873 constraint(ALLOC_IN_RC(int_r4_reg));
4874 match(RegI);
4875 match(iRegINoSp);
4876 op_cost(0);
4877 format %{ %}
4878 interface(REG_INTER);
4879 %}
4880
4881
4882 // Pointer Register Operands
4883 // Narrow Pointer Register
4884 operand iRegN()
4885 %{
4886 constraint(ALLOC_IN_RC(any_reg32));
4887 match(RegN);
4888 match(iRegNNoSp);
4889 op_cost(0);
4890 format %{ %}
4891 interface(REG_INTER);
4892 %}
4893
4894 // Integer 64 bit Register not Special
4895 operand iRegNNoSp()
4896 %{
4897 constraint(ALLOC_IN_RC(no_special_reg32));
4898 match(RegN);
4899 op_cost(0);
4900 format %{ %}
4901 interface(REG_INTER);
4902 %}
4903
4904 // Float Register
4905 // Float register operands
4906 operand vRegF()
4907 %{
4908 constraint(ALLOC_IN_RC(float_reg));
4909 match(RegF);
4910
4911 op_cost(0);
4912 format %{ %}
4913 interface(REG_INTER);
4914 %}
4915
4916 // Double Register
4917 // Double register operands
4918 operand vRegD()
4919 %{
4920 constraint(ALLOC_IN_RC(double_reg));
4921 match(RegD);
4922
4923 op_cost(0);
4924 format %{ %}
4925 interface(REG_INTER);
4926 %}
4927
4928 // Generic vector class. This will be used for
4929 // all vector operands, including NEON and SVE.
4930 operand vReg()
4931 %{
4932 constraint(ALLOC_IN_RC(dynamic));
4933 match(VecA);
4934 match(VecD);
4935 match(VecX);
4936
4937 op_cost(0);
4938 format %{ %}
4939 interface(REG_INTER);
4940 %}
4941
4942 operand vReg_V10()
4943 %{
4944 constraint(ALLOC_IN_RC(v10_veca_reg));
4945 match(vReg);
4946
4947 op_cost(0);
4948 format %{ %}
4949 interface(REG_INTER);
4950 %}
4951
4952 operand vReg_V11()
4953 %{
4954 constraint(ALLOC_IN_RC(v11_veca_reg));
4955 match(vReg);
4956
4957 op_cost(0);
4958 format %{ %}
4959 interface(REG_INTER);
4960 %}
4961
4962 operand vReg_V12()
4963 %{
4964 constraint(ALLOC_IN_RC(v12_veca_reg));
4965 match(vReg);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 operand vReg_V13()
4973 %{
4974 constraint(ALLOC_IN_RC(v13_veca_reg));
4975 match(vReg);
4976
4977 op_cost(0);
4978 format %{ %}
4979 interface(REG_INTER);
4980 %}
4981
4982 operand vReg_V17()
4983 %{
4984 constraint(ALLOC_IN_RC(v17_veca_reg));
4985 match(vReg);
4986
4987 op_cost(0);
4988 format %{ %}
4989 interface(REG_INTER);
4990 %}
4991
4992 operand vReg_V18()
4993 %{
4994 constraint(ALLOC_IN_RC(v18_veca_reg));
4995 match(vReg);
4996
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 operand vReg_V23()
5003 %{
5004 constraint(ALLOC_IN_RC(v23_veca_reg));
5005 match(vReg);
5006
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vReg_V24()
5013 %{
5014 constraint(ALLOC_IN_RC(v24_veca_reg));
5015 match(vReg);
5016
5017 op_cost(0);
5018 format %{ %}
5019 interface(REG_INTER);
5020 %}
5021
5022 operand vecA()
5023 %{
5024 constraint(ALLOC_IN_RC(vectora_reg));
5025 match(VecA);
5026
5027 op_cost(0);
5028 format %{ %}
5029 interface(REG_INTER);
5030 %}
5031
5032 operand vecD()
5033 %{
5034 constraint(ALLOC_IN_RC(vectord_reg));
5035 match(VecD);
5036
5037 op_cost(0);
5038 format %{ %}
5039 interface(REG_INTER);
5040 %}
5041
5042 operand vecX()
5043 %{
5044 constraint(ALLOC_IN_RC(vectorx_reg));
5045 match(VecX);
5046
5047 op_cost(0);
5048 format %{ %}
5049 interface(REG_INTER);
5050 %}
5051
5052 operand vRegD_V0()
5053 %{
5054 constraint(ALLOC_IN_RC(v0_reg));
5055 match(RegD);
5056 op_cost(0);
5057 format %{ %}
5058 interface(REG_INTER);
5059 %}
5060
5061 operand vRegD_V1()
5062 %{
5063 constraint(ALLOC_IN_RC(v1_reg));
5064 match(RegD);
5065 op_cost(0);
5066 format %{ %}
5067 interface(REG_INTER);
5068 %}
5069
5070 operand vRegD_V2()
5071 %{
5072 constraint(ALLOC_IN_RC(v2_reg));
5073 match(RegD);
5074 op_cost(0);
5075 format %{ %}
5076 interface(REG_INTER);
5077 %}
5078
5079 operand vRegD_V3()
5080 %{
5081 constraint(ALLOC_IN_RC(v3_reg));
5082 match(RegD);
5083 op_cost(0);
5084 format %{ %}
5085 interface(REG_INTER);
5086 %}
5087
5088 operand vRegD_V4()
5089 %{
5090 constraint(ALLOC_IN_RC(v4_reg));
5091 match(RegD);
5092 op_cost(0);
5093 format %{ %}
5094 interface(REG_INTER);
5095 %}
5096
5097 operand vRegD_V5()
5098 %{
5099 constraint(ALLOC_IN_RC(v5_reg));
5100 match(RegD);
5101 op_cost(0);
5102 format %{ %}
5103 interface(REG_INTER);
5104 %}
5105
5106 operand vRegD_V6()
5107 %{
5108 constraint(ALLOC_IN_RC(v6_reg));
5109 match(RegD);
5110 op_cost(0);
5111 format %{ %}
5112 interface(REG_INTER);
5113 %}
5114
5115 operand vRegD_V7()
5116 %{
5117 constraint(ALLOC_IN_RC(v7_reg));
5118 match(RegD);
5119 op_cost(0);
5120 format %{ %}
5121 interface(REG_INTER);
5122 %}
5123
5124 operand vRegD_V12()
5125 %{
5126 constraint(ALLOC_IN_RC(v12_reg));
5127 match(RegD);
5128 op_cost(0);
5129 format %{ %}
5130 interface(REG_INTER);
5131 %}
5132
5133 operand vRegD_V13()
5134 %{
5135 constraint(ALLOC_IN_RC(v13_reg));
5136 match(RegD);
5137 op_cost(0);
5138 format %{ %}
5139 interface(REG_INTER);
5140 %}
5141
5142 operand pReg()
5143 %{
5144 constraint(ALLOC_IN_RC(pr_reg));
5145 match(RegVectMask);
5146 match(pRegGov);
5147 op_cost(0);
5148 format %{ %}
5149 interface(REG_INTER);
5150 %}
5151
5152 operand pRegGov()
5153 %{
5154 constraint(ALLOC_IN_RC(gov_pr));
5155 match(RegVectMask);
5156 match(pReg);
5157 op_cost(0);
5158 format %{ %}
5159 interface(REG_INTER);
5160 %}
5161
5162 operand pRegGov_P0()
5163 %{
5164 constraint(ALLOC_IN_RC(p0_reg));
5165 match(RegVectMask);
5166 op_cost(0);
5167 format %{ %}
5168 interface(REG_INTER);
5169 %}
5170
5171 operand pRegGov_P1()
5172 %{
5173 constraint(ALLOC_IN_RC(p1_reg));
5174 match(RegVectMask);
5175 op_cost(0);
5176 format %{ %}
5177 interface(REG_INTER);
5178 %}
5179
5180 // Flags register, used as output of signed compare instructions
5181
5182 // note that on AArch64 we also use this register as the output for
5183 // for floating point compare instructions (CmpF CmpD). this ensures
5184 // that ordered inequality tests use GT, GE, LT or LE none of which
5185 // pass through cases where the result is unordered i.e. one or both
5186 // inputs to the compare is a NaN. this means that the ideal code can
5187 // replace e.g. a GT with an LE and not end up capturing the NaN case
5188 // (where the comparison should always fail). EQ and NE tests are
5189 // always generated in ideal code so that unordered folds into the NE
5190 // case, matching the behaviour of AArch64 NE.
5191 //
5192 // This differs from x86 where the outputs of FP compares use a
5193 // special FP flags registers and where compares based on this
5194 // register are distinguished into ordered inequalities (cmpOpUCF) and
5195 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5196 // to explicitly handle the unordered case in branches. x86 also has
5197 // to include extra CMoveX rules to accept a cmpOpUCF input.
5198
5199 operand rFlagsReg()
5200 %{
5201 constraint(ALLOC_IN_RC(int_flags));
5202 match(RegFlags);
5203
5204 op_cost(0);
5205 format %{ "RFLAGS" %}
5206 interface(REG_INTER);
5207 %}
5208
5209 // Flags register, used as output of unsigned compare instructions
5210 operand rFlagsRegU()
5211 %{
5212 constraint(ALLOC_IN_RC(int_flags));
5213 match(RegFlags);
5214
5215 op_cost(0);
5216 format %{ "RFLAGSU" %}
5217 interface(REG_INTER);
5218 %}
5219
5220 // Special Registers
5221
5222 // Method Register
5223 operand inline_cache_RegP(iRegP reg)
5224 %{
5225 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5226 match(reg);
5227 match(iRegPNoSp);
5228 op_cost(0);
5229 format %{ %}
5230 interface(REG_INTER);
5231 %}
5232
5233 // Thread Register
5234 operand thread_RegP(iRegP reg)
5235 %{
5236 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5237 match(reg);
5238 op_cost(0);
5239 format %{ %}
5240 interface(REG_INTER);
5241 %}
5242
5243 //----------Memory Operands----------------------------------------------------
5244
5245 operand indirect(iRegP reg)
5246 %{
5247 constraint(ALLOC_IN_RC(ptr_reg));
5248 match(reg);
5249 op_cost(0);
5250 format %{ "[$reg]" %}
5251 interface(MEMORY_INTER) %{
5252 base($reg);
5253 index(0xffffffff);
5254 scale(0x0);
5255 disp(0x0);
5256 %}
5257 %}
5258
5259 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5260 %{
5261 constraint(ALLOC_IN_RC(ptr_reg));
5262 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5263 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5264 op_cost(0);
5265 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5266 interface(MEMORY_INTER) %{
5267 base($reg);
5268 index($ireg);
5269 scale($scale);
5270 disp(0x0);
5271 %}
5272 %}
5273
5274 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5275 %{
5276 constraint(ALLOC_IN_RC(ptr_reg));
5277 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5278 match(AddP reg (LShiftL lreg scale));
5279 op_cost(0);
5280 format %{ "$reg, $lreg lsl($scale)" %}
5281 interface(MEMORY_INTER) %{
5282 base($reg);
5283 index($lreg);
5284 scale($scale);
5285 disp(0x0);
5286 %}
5287 %}
5288
5289 operand indIndexI2L(iRegP reg, iRegI ireg)
5290 %{
5291 constraint(ALLOC_IN_RC(ptr_reg));
5292 match(AddP reg (ConvI2L ireg));
5293 op_cost(0);
5294 format %{ "$reg, $ireg, 0, I2L" %}
5295 interface(MEMORY_INTER) %{
5296 base($reg);
5297 index($ireg);
5298 scale(0x0);
5299 disp(0x0);
5300 %}
5301 %}
5302
5303 operand indIndex(iRegP reg, iRegL lreg)
5304 %{
5305 constraint(ALLOC_IN_RC(ptr_reg));
5306 match(AddP reg lreg);
5307 op_cost(0);
5308 format %{ "$reg, $lreg" %}
5309 interface(MEMORY_INTER) %{
5310 base($reg);
5311 index($lreg);
5312 scale(0x0);
5313 disp(0x0);
5314 %}
5315 %}
5316
5317 operand indOffI1(iRegP reg, immIOffset1 off)
5318 %{
5319 constraint(ALLOC_IN_RC(ptr_reg));
5320 match(AddP reg off);
5321 op_cost(0);
5322 format %{ "[$reg, $off]" %}
5323 interface(MEMORY_INTER) %{
5324 base($reg);
5325 index(0xffffffff);
5326 scale(0x0);
5327 disp($off);
5328 %}
5329 %}
5330
5331 operand indOffI2(iRegP reg, immIOffset2 off)
5332 %{
5333 constraint(ALLOC_IN_RC(ptr_reg));
5334 match(AddP reg off);
5335 op_cost(0);
5336 format %{ "[$reg, $off]" %}
5337 interface(MEMORY_INTER) %{
5338 base($reg);
5339 index(0xffffffff);
5340 scale(0x0);
5341 disp($off);
5342 %}
5343 %}
5344
5345 operand indOffI4(iRegP reg, immIOffset4 off)
5346 %{
5347 constraint(ALLOC_IN_RC(ptr_reg));
5348 match(AddP reg off);
5349 op_cost(0);
5350 format %{ "[$reg, $off]" %}
5351 interface(MEMORY_INTER) %{
5352 base($reg);
5353 index(0xffffffff);
5354 scale(0x0);
5355 disp($off);
5356 %}
5357 %}
5358
5359 operand indOffI8(iRegP reg, immIOffset8 off)
5360 %{
5361 constraint(ALLOC_IN_RC(ptr_reg));
5362 match(AddP reg off);
5363 op_cost(0);
5364 format %{ "[$reg, $off]" %}
5365 interface(MEMORY_INTER) %{
5366 base($reg);
5367 index(0xffffffff);
5368 scale(0x0);
5369 disp($off);
5370 %}
5371 %}
5372
5373 operand indOffI16(iRegP reg, immIOffset16 off)
5374 %{
5375 constraint(ALLOC_IN_RC(ptr_reg));
5376 match(AddP reg off);
5377 op_cost(0);
5378 format %{ "[$reg, $off]" %}
5379 interface(MEMORY_INTER) %{
5380 base($reg);
5381 index(0xffffffff);
5382 scale(0x0);
5383 disp($off);
5384 %}
5385 %}
5386
5387 operand indOffL1(iRegP reg, immLoffset1 off)
5388 %{
5389 constraint(ALLOC_IN_RC(ptr_reg));
5390 match(AddP reg off);
5391 op_cost(0);
5392 format %{ "[$reg, $off]" %}
5393 interface(MEMORY_INTER) %{
5394 base($reg);
5395 index(0xffffffff);
5396 scale(0x0);
5397 disp($off);
5398 %}
5399 %}
5400
5401 operand indOffL2(iRegP reg, immLoffset2 off)
5402 %{
5403 constraint(ALLOC_IN_RC(ptr_reg));
5404 match(AddP reg off);
5405 op_cost(0);
5406 format %{ "[$reg, $off]" %}
5407 interface(MEMORY_INTER) %{
5408 base($reg);
5409 index(0xffffffff);
5410 scale(0x0);
5411 disp($off);
5412 %}
5413 %}
5414
5415 operand indOffL4(iRegP reg, immLoffset4 off)
5416 %{
5417 constraint(ALLOC_IN_RC(ptr_reg));
5418 match(AddP reg off);
5419 op_cost(0);
5420 format %{ "[$reg, $off]" %}
5421 interface(MEMORY_INTER) %{
5422 base($reg);
5423 index(0xffffffff);
5424 scale(0x0);
5425 disp($off);
5426 %}
5427 %}
5428
5429 operand indOffL8(iRegP reg, immLoffset8 off)
5430 %{
5431 constraint(ALLOC_IN_RC(ptr_reg));
5432 match(AddP reg off);
5433 op_cost(0);
5434 format %{ "[$reg, $off]" %}
5435 interface(MEMORY_INTER) %{
5436 base($reg);
5437 index(0xffffffff);
5438 scale(0x0);
5439 disp($off);
5440 %}
5441 %}
5442
5443 operand indOffL16(iRegP reg, immLoffset16 off)
5444 %{
5445 constraint(ALLOC_IN_RC(ptr_reg));
5446 match(AddP reg off);
5447 op_cost(0);
5448 format %{ "[$reg, $off]" %}
5449 interface(MEMORY_INTER) %{
5450 base($reg);
5451 index(0xffffffff);
5452 scale(0x0);
5453 disp($off);
5454 %}
5455 %}
5456
5457 operand indirectX2P(iRegL reg)
5458 %{
5459 constraint(ALLOC_IN_RC(ptr_reg));
5460 match(CastX2P reg);
5461 op_cost(0);
5462 format %{ "[$reg]\t# long -> ptr" %}
5463 interface(MEMORY_INTER) %{
5464 base($reg);
5465 index(0xffffffff);
5466 scale(0x0);
5467 disp(0x0);
5468 %}
5469 %}
5470
5471 operand indOffX2P(iRegL reg, immLOffset off)
5472 %{
5473 constraint(ALLOC_IN_RC(ptr_reg));
5474 match(AddP (CastX2P reg) off);
5475 op_cost(0);
5476 format %{ "[$reg, $off]\t# long -> ptr" %}
5477 interface(MEMORY_INTER) %{
5478 base($reg);
5479 index(0xffffffff);
5480 scale(0x0);
5481 disp($off);
5482 %}
5483 %}
5484
5485 operand indirectN(iRegN reg)
5486 %{
5487 predicate(CompressedOops::shift() == 0);
5488 constraint(ALLOC_IN_RC(ptr_reg));
5489 match(DecodeN reg);
5490 op_cost(0);
5491 format %{ "[$reg]\t# narrow" %}
5492 interface(MEMORY_INTER) %{
5493 base($reg);
5494 index(0xffffffff);
5495 scale(0x0);
5496 disp(0x0);
5497 %}
5498 %}
5499
5500 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5501 %{
5502 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5503 constraint(ALLOC_IN_RC(ptr_reg));
5504 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5505 op_cost(0);
5506 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5507 interface(MEMORY_INTER) %{
5508 base($reg);
5509 index($ireg);
5510 scale($scale);
5511 disp(0x0);
5512 %}
5513 %}
5514
5515 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5516 %{
5517 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5518 constraint(ALLOC_IN_RC(ptr_reg));
5519 match(AddP (DecodeN reg) (LShiftL lreg scale));
5520 op_cost(0);
5521 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5522 interface(MEMORY_INTER) %{
5523 base($reg);
5524 index($lreg);
5525 scale($scale);
5526 disp(0x0);
5527 %}
5528 %}
5529
5530 operand indIndexI2LN(iRegN reg, iRegI ireg)
5531 %{
5532 predicate(CompressedOops::shift() == 0);
5533 constraint(ALLOC_IN_RC(ptr_reg));
5534 match(AddP (DecodeN reg) (ConvI2L ireg));
5535 op_cost(0);
5536 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5537 interface(MEMORY_INTER) %{
5538 base($reg);
5539 index($ireg);
5540 scale(0x0);
5541 disp(0x0);
5542 %}
5543 %}
5544
5545 operand indIndexN(iRegN reg, iRegL lreg)
5546 %{
5547 predicate(CompressedOops::shift() == 0);
5548 constraint(ALLOC_IN_RC(ptr_reg));
5549 match(AddP (DecodeN reg) lreg);
5550 op_cost(0);
5551 format %{ "$reg, $lreg\t# narrow" %}
5552 interface(MEMORY_INTER) %{
5553 base($reg);
5554 index($lreg);
5555 scale(0x0);
5556 disp(0x0);
5557 %}
5558 %}
5559
5560 operand indOffIN(iRegN reg, immIOffset off)
5561 %{
5562 predicate(CompressedOops::shift() == 0);
5563 constraint(ALLOC_IN_RC(ptr_reg));
5564 match(AddP (DecodeN reg) off);
5565 op_cost(0);
5566 format %{ "[$reg, $off]\t# narrow" %}
5567 interface(MEMORY_INTER) %{
5568 base($reg);
5569 index(0xffffffff);
5570 scale(0x0);
5571 disp($off);
5572 %}
5573 %}
5574
5575 operand indOffLN(iRegN reg, immLOffset off)
5576 %{
5577 predicate(CompressedOops::shift() == 0);
5578 constraint(ALLOC_IN_RC(ptr_reg));
5579 match(AddP (DecodeN reg) off);
5580 op_cost(0);
5581 format %{ "[$reg, $off]\t# narrow" %}
5582 interface(MEMORY_INTER) %{
5583 base($reg);
5584 index(0xffffffff);
5585 scale(0x0);
5586 disp($off);
5587 %}
5588 %}
5589
5590
5591 //----------Special Memory Operands--------------------------------------------
5592 // Stack Slot Operand - This operand is used for loading and storing temporary
5593 // values on the stack where a match requires a value to
5594 // flow through memory.
5595 operand stackSlotP(sRegP reg)
5596 %{
5597 constraint(ALLOC_IN_RC(stack_slots));
5598 op_cost(100);
5599 // No match rule because this operand is only generated in matching
5600 // match(RegP);
5601 format %{ "[$reg]" %}
5602 interface(MEMORY_INTER) %{
5603 base(0x1e); // RSP
5604 index(0x0); // No Index
5605 scale(0x0); // No Scale
5606 disp($reg); // Stack Offset
5607 %}
5608 %}
5609
5610 operand stackSlotI(sRegI reg)
5611 %{
5612 constraint(ALLOC_IN_RC(stack_slots));
5613 // No match rule because this operand is only generated in matching
5614 // match(RegI);
5615 format %{ "[$reg]" %}
5616 interface(MEMORY_INTER) %{
5617 base(0x1e); // RSP
5618 index(0x0); // No Index
5619 scale(0x0); // No Scale
5620 disp($reg); // Stack Offset
5621 %}
5622 %}
5623
5624 operand stackSlotF(sRegF reg)
5625 %{
5626 constraint(ALLOC_IN_RC(stack_slots));
5627 // No match rule because this operand is only generated in matching
5628 // match(RegF);
5629 format %{ "[$reg]" %}
5630 interface(MEMORY_INTER) %{
5631 base(0x1e); // RSP
5632 index(0x0); // No Index
5633 scale(0x0); // No Scale
5634 disp($reg); // Stack Offset
5635 %}
5636 %}
5637
5638 operand stackSlotD(sRegD reg)
5639 %{
5640 constraint(ALLOC_IN_RC(stack_slots));
5641 // No match rule because this operand is only generated in matching
5642 // match(RegD);
5643 format %{ "[$reg]" %}
5644 interface(MEMORY_INTER) %{
5645 base(0x1e); // RSP
5646 index(0x0); // No Index
5647 scale(0x0); // No Scale
5648 disp($reg); // Stack Offset
5649 %}
5650 %}
5651
5652 operand stackSlotL(sRegL reg)
5653 %{
5654 constraint(ALLOC_IN_RC(stack_slots));
5655 // No match rule because this operand is only generated in matching
5656 // match(RegL);
5657 format %{ "[$reg]" %}
5658 interface(MEMORY_INTER) %{
5659 base(0x1e); // RSP
5660 index(0x0); // No Index
5661 scale(0x0); // No Scale
5662 disp($reg); // Stack Offset
5663 %}
5664 %}
5665
5666 // Operands for expressing Control Flow
5667 // NOTE: Label is a predefined operand which should not be redefined in
5668 // the AD file. It is generically handled within the ADLC.
5669
5670 //----------Conditional Branch Operands----------------------------------------
5671 // Comparison Op - This is the operation of the comparison, and is limited to
5672 // the following set of codes:
5673 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5674 //
5675 // Other attributes of the comparison, such as unsignedness, are specified
5676 // by the comparison instruction that sets a condition code flags register.
5677 // That result is represented by a flags operand whose subtype is appropriate
5678 // to the unsignedness (etc.) of the comparison.
5679 //
5680 // Later, the instruction which matches both the Comparison Op (a Bool) and
5681 // the flags (produced by the Cmp) specifies the coding of the comparison op
5682 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5683
5684 // used for signed integral comparisons and fp comparisons
5685
5686 operand cmpOp()
5687 %{
5688 match(Bool);
5689
5690 format %{ "" %}
5691 interface(COND_INTER) %{
5692 equal(0x0, "eq");
5693 not_equal(0x1, "ne");
5694 less(0xb, "lt");
5695 greater_equal(0xa, "ge");
5696 less_equal(0xd, "le");
5697 greater(0xc, "gt");
5698 overflow(0x6, "vs");
5699 no_overflow(0x7, "vc");
5700 %}
5701 %}
5702
5703 // used for unsigned integral comparisons
5704
5705 operand cmpOpU()
5706 %{
5707 match(Bool);
5708
5709 format %{ "" %}
5710 interface(COND_INTER) %{
5711 equal(0x0, "eq");
5712 not_equal(0x1, "ne");
5713 less(0x3, "lo");
5714 greater_equal(0x2, "hs");
5715 less_equal(0x9, "ls");
5716 greater(0x8, "hi");
5717 overflow(0x6, "vs");
5718 no_overflow(0x7, "vc");
5719 %}
5720 %}
5721
5722 // used for certain integral comparisons which can be
5723 // converted to cbxx or tbxx instructions
5724
5725 operand cmpOpEqNe()
5726 %{
5727 match(Bool);
5728 op_cost(0);
5729 predicate(n->as_Bool()->_test._test == BoolTest::ne
5730 || n->as_Bool()->_test._test == BoolTest::eq);
5731
5732 format %{ "" %}
5733 interface(COND_INTER) %{
5734 equal(0x0, "eq");
5735 not_equal(0x1, "ne");
5736 less(0xb, "lt");
5737 greater_equal(0xa, "ge");
5738 less_equal(0xd, "le");
5739 greater(0xc, "gt");
5740 overflow(0x6, "vs");
5741 no_overflow(0x7, "vc");
5742 %}
5743 %}
5744
5745 // used for certain integral comparisons which can be
5746 // converted to cbxx or tbxx instructions
5747
5748 operand cmpOpLtGe()
5749 %{
5750 match(Bool);
5751 op_cost(0);
5752
5753 predicate(n->as_Bool()->_test._test == BoolTest::lt
5754 || n->as_Bool()->_test._test == BoolTest::ge);
5755
5756 format %{ "" %}
5757 interface(COND_INTER) %{
5758 equal(0x0, "eq");
5759 not_equal(0x1, "ne");
5760 less(0xb, "lt");
5761 greater_equal(0xa, "ge");
5762 less_equal(0xd, "le");
5763 greater(0xc, "gt");
5764 overflow(0x6, "vs");
5765 no_overflow(0x7, "vc");
5766 %}
5767 %}
5768
5769 // used for certain unsigned integral comparisons which can be
5770 // converted to cbxx or tbxx instructions
5771
5772 operand cmpOpUEqNeLeGt()
5773 %{
5774 match(Bool);
5775 op_cost(0);
5776
5777 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5778 n->as_Bool()->_test._test == BoolTest::ne ||
5779 n->as_Bool()->_test._test == BoolTest::le ||
5780 n->as_Bool()->_test._test == BoolTest::gt);
5781
5782 format %{ "" %}
5783 interface(COND_INTER) %{
5784 equal(0x0, "eq");
5785 not_equal(0x1, "ne");
5786 less(0x3, "lo");
5787 greater_equal(0x2, "hs");
5788 less_equal(0x9, "ls");
5789 greater(0x8, "hi");
5790 overflow(0x6, "vs");
5791 no_overflow(0x7, "vc");
5792 %}
5793 %}
5794
5795 // Special operand allowing long args to int ops to be truncated for free
5796
5797 operand iRegL2I(iRegL reg) %{
5798
5799 op_cost(0);
5800
5801 match(ConvL2I reg);
5802
5803 format %{ "l2i($reg)" %}
5804
5805 interface(REG_INTER)
5806 %}
5807
5808 operand iRegL2P(iRegL reg) %{
5809
5810 op_cost(0);
5811
5812 match(CastX2P reg);
5813
5814 format %{ "l2p($reg)" %}
5815
5816 interface(REG_INTER)
5817 %}
5818
5819 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5820 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5821 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5822 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5823
5824 //----------OPERAND CLASSES----------------------------------------------------
5825 // Operand Classes are groups of operands that are used as to simplify
5826 // instruction definitions by not requiring the AD writer to specify
5827 // separate instructions for every form of operand when the
5828 // instruction accepts multiple operand types with the same basic
5829 // encoding and format. The classic case of this is memory operands.
5830
5831 // memory is used to define read/write location for load/store
5832 // instruction defs. we can turn a memory op into an Address
5833
5834 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5835 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5836
5837 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5838 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5839
5840 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5841 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5842
5843 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5844 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5845
5846 // All of the memory operands. For the pipeline description.
5847 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5848 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5849 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5850
5851
5852 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5853 // operations. it allows the src to be either an iRegI or a (ConvL2I
5854 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5855 // can be elided because the 32-bit instruction will just employ the
5856 // lower 32 bits anyway.
5857 //
5858 // n.b. this does not elide all L2I conversions. if the truncated
5859 // value is consumed by more than one operation then the ConvL2I
5860 // cannot be bundled into the consuming nodes so an l2i gets planted
5861 // (actually a movw $dst $src) and the downstream instructions consume
5862 // the result of the l2i as an iRegI input. That's a shame since the
5863 // movw is actually redundant but its not too costly.
5864
5865 opclass iRegIorL2I(iRegI, iRegL2I);
5866 opclass iRegPorL2P(iRegP, iRegL2P);
5867
5868 //----------PIPELINE-----------------------------------------------------------
5869 // Rules which define the behavior of the target architectures pipeline.
5870
5871 // For specific pipelines, eg A53, define the stages of that pipeline
5872 //pipe_desc(ISS, EX1, EX2, WR);
5873 #define ISS S0
5874 #define EX1 S1
5875 #define EX2 S2
5876 #define WR S3
5877
5878 // Integer ALU reg operation
5879 pipeline %{
5880
5881 attributes %{
5882 // ARM instructions are of fixed length
5883 fixed_size_instructions; // Fixed size instructions TODO does
5884 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5885 // ARM instructions come in 32-bit word units
5886 instruction_unit_size = 4; // An instruction is 4 bytes long
5887 instruction_fetch_unit_size = 64; // The processor fetches one line
5888 instruction_fetch_units = 1; // of 64 bytes
5889 %}
5890
5891 // We don't use an actual pipeline model so don't care about resources
5892 // or description. we do use pipeline classes to introduce fixed
5893 // latencies
5894
5895 //----------RESOURCES----------------------------------------------------------
5896 // Resources are the functional units available to the machine
5897
5898 resources( INS0, INS1, INS01 = INS0 | INS1,
5899 ALU0, ALU1, ALU = ALU0 | ALU1,
5900 MAC,
5901 DIV,
5902 BRANCH,
5903 LDST,
5904 NEON_FP);
5905
5906 //----------PIPELINE DESCRIPTION-----------------------------------------------
5907 // Pipeline Description specifies the stages in the machine's pipeline
5908
5909 // Define the pipeline as a generic 6 stage pipeline
5910 pipe_desc(S0, S1, S2, S3, S4, S5);
5911
5912 //----------PIPELINE CLASSES---------------------------------------------------
5913 // Pipeline Classes describe the stages in which input and output are
5914 // referenced by the hardware pipeline.
5915
5916 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5917 %{
5918 single_instruction;
5919 src1 : S1(read);
5920 src2 : S2(read);
5921 dst : S5(write);
5922 INS01 : ISS;
5923 NEON_FP : S5;
5924 %}
5925
5926 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5927 %{
5928 single_instruction;
5929 src1 : S1(read);
5930 src2 : S2(read);
5931 dst : S5(write);
5932 INS01 : ISS;
5933 NEON_FP : S5;
5934 %}
5935
5936 pipe_class fp_uop_s(vRegF dst, vRegF src)
5937 %{
5938 single_instruction;
5939 src : S1(read);
5940 dst : S5(write);
5941 INS01 : ISS;
5942 NEON_FP : S5;
5943 %}
5944
5945 pipe_class fp_uop_d(vRegD dst, vRegD src)
5946 %{
5947 single_instruction;
5948 src : S1(read);
5949 dst : S5(write);
5950 INS01 : ISS;
5951 NEON_FP : S5;
5952 %}
5953
5954 pipe_class fp_d2f(vRegF dst, vRegD src)
5955 %{
5956 single_instruction;
5957 src : S1(read);
5958 dst : S5(write);
5959 INS01 : ISS;
5960 NEON_FP : S5;
5961 %}
5962
5963 pipe_class fp_f2d(vRegD dst, vRegF src)
5964 %{
5965 single_instruction;
5966 src : S1(read);
5967 dst : S5(write);
5968 INS01 : ISS;
5969 NEON_FP : S5;
5970 %}
5971
5972 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5973 %{
5974 single_instruction;
5975 src : S1(read);
5976 dst : S5(write);
5977 INS01 : ISS;
5978 NEON_FP : S5;
5979 %}
5980
5981 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5982 %{
5983 single_instruction;
5984 src : S1(read);
5985 dst : S5(write);
5986 INS01 : ISS;
5987 NEON_FP : S5;
5988 %}
5989
5990 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5991 %{
5992 single_instruction;
5993 src : S1(read);
5994 dst : S5(write);
5995 INS01 : ISS;
5996 NEON_FP : S5;
5997 %}
5998
5999 pipe_class fp_l2f(vRegF dst, iRegL src)
6000 %{
6001 single_instruction;
6002 src : S1(read);
6003 dst : S5(write);
6004 INS01 : ISS;
6005 NEON_FP : S5;
6006 %}
6007
6008 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6009 %{
6010 single_instruction;
6011 src : S1(read);
6012 dst : S5(write);
6013 INS01 : ISS;
6014 NEON_FP : S5;
6015 %}
6016
6017 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6018 %{
6019 single_instruction;
6020 src : S1(read);
6021 dst : S5(write);
6022 INS01 : ISS;
6023 NEON_FP : S5;
6024 %}
6025
6026 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6027 %{
6028 single_instruction;
6029 src : S1(read);
6030 dst : S5(write);
6031 INS01 : ISS;
6032 NEON_FP : S5;
6033 %}
6034
6035 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6036 %{
6037 single_instruction;
6038 src : S1(read);
6039 dst : S5(write);
6040 INS01 : ISS;
6041 NEON_FP : S5;
6042 %}
6043
6044 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6045 %{
6046 single_instruction;
6047 src1 : S1(read);
6048 src2 : S2(read);
6049 dst : S5(write);
6050 INS0 : ISS;
6051 NEON_FP : S5;
6052 %}
6053
6054 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6055 %{
6056 single_instruction;
6057 src1 : S1(read);
6058 src2 : S2(read);
6059 dst : S5(write);
6060 INS0 : ISS;
6061 NEON_FP : S5;
6062 %}
6063
6064 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6065 %{
6066 single_instruction;
6067 cr : S1(read);
6068 src1 : S1(read);
6069 src2 : S1(read);
6070 dst : S3(write);
6071 INS01 : ISS;
6072 NEON_FP : S3;
6073 %}
6074
6075 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6076 %{
6077 single_instruction;
6078 cr : S1(read);
6079 src1 : S1(read);
6080 src2 : S1(read);
6081 dst : S3(write);
6082 INS01 : ISS;
6083 NEON_FP : S3;
6084 %}
6085
6086 pipe_class fp_imm_s(vRegF dst)
6087 %{
6088 single_instruction;
6089 dst : S3(write);
6090 INS01 : ISS;
6091 NEON_FP : S3;
6092 %}
6093
6094 pipe_class fp_imm_d(vRegD dst)
6095 %{
6096 single_instruction;
6097 dst : S3(write);
6098 INS01 : ISS;
6099 NEON_FP : S3;
6100 %}
6101
6102 pipe_class fp_load_constant_s(vRegF dst)
6103 %{
6104 single_instruction;
6105 dst : S4(write);
6106 INS01 : ISS;
6107 NEON_FP : S4;
6108 %}
6109
6110 pipe_class fp_load_constant_d(vRegD dst)
6111 %{
6112 single_instruction;
6113 dst : S4(write);
6114 INS01 : ISS;
6115 NEON_FP : S4;
6116 %}
6117
6118 //------- Integer ALU operations --------------------------
6119
6120 // Integer ALU reg-reg operation
6121 // Operands needed in EX1, result generated in EX2
6122 // Eg. ADD x0, x1, x2
6123 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6124 %{
6125 single_instruction;
6126 dst : EX2(write);
6127 src1 : EX1(read);
6128 src2 : EX1(read);
6129 INS01 : ISS; // Dual issue as instruction 0 or 1
6130 ALU : EX2;
6131 %}
6132
6133 // Integer ALU reg-reg operation with constant shift
6134 // Shifted register must be available in LATE_ISS instead of EX1
6135 // Eg. ADD x0, x1, x2, LSL #2
6136 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6137 %{
6138 single_instruction;
6139 dst : EX2(write);
6140 src1 : EX1(read);
6141 src2 : ISS(read);
6142 INS01 : ISS;
6143 ALU : EX2;
6144 %}
6145
6146 // Integer ALU reg operation with constant shift
6147 // Eg. LSL x0, x1, #shift
6148 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6149 %{
6150 single_instruction;
6151 dst : EX2(write);
6152 src1 : ISS(read);
6153 INS01 : ISS;
6154 ALU : EX2;
6155 %}
6156
6157 // Integer ALU reg-reg operation with variable shift
6158 // Both operands must be available in LATE_ISS instead of EX1
6159 // Result is available in EX1 instead of EX2
6160 // Eg. LSLV x0, x1, x2
6161 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6162 %{
6163 single_instruction;
6164 dst : EX1(write);
6165 src1 : ISS(read);
6166 src2 : ISS(read);
6167 INS01 : ISS;
6168 ALU : EX1;
6169 %}
6170
6171 // Integer ALU reg-reg operation with extract
6172 // As for _vshift above, but result generated in EX2
6173 // Eg. EXTR x0, x1, x2, #N
6174 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6175 %{
6176 single_instruction;
6177 dst : EX2(write);
6178 src1 : ISS(read);
6179 src2 : ISS(read);
6180 INS1 : ISS; // Can only dual issue as Instruction 1
6181 ALU : EX1;
6182 %}
6183
6184 // Integer ALU reg operation
6185 // Eg. NEG x0, x1
6186 pipe_class ialu_reg(iRegI dst, iRegI src)
6187 %{
6188 single_instruction;
6189 dst : EX2(write);
6190 src : EX1(read);
6191 INS01 : ISS;
6192 ALU : EX2;
6193 %}
6194
6195 // Integer ALU reg mmediate operation
6196 // Eg. ADD x0, x1, #N
6197 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6198 %{
6199 single_instruction;
6200 dst : EX2(write);
6201 src1 : EX1(read);
6202 INS01 : ISS;
6203 ALU : EX2;
6204 %}
6205
6206 // Integer ALU immediate operation (no source operands)
6207 // Eg. MOV x0, #N
6208 pipe_class ialu_imm(iRegI dst)
6209 %{
6210 single_instruction;
6211 dst : EX1(write);
6212 INS01 : ISS;
6213 ALU : EX1;
6214 %}
6215
6216 //------- Compare operation -------------------------------
6217
6218 // Compare reg-reg
6219 // Eg. CMP x0, x1
6220 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6221 %{
6222 single_instruction;
6223 // fixed_latency(16);
6224 cr : EX2(write);
6225 op1 : EX1(read);
6226 op2 : EX1(read);
6227 INS01 : ISS;
6228 ALU : EX2;
6229 %}
6230
6231 // Compare reg-reg
6232 // Eg. CMP x0, #N
6233 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6234 %{
6235 single_instruction;
6236 // fixed_latency(16);
6237 cr : EX2(write);
6238 op1 : EX1(read);
6239 INS01 : ISS;
6240 ALU : EX2;
6241 %}
6242
6243 //------- Conditional instructions ------------------------
6244
6245 // Conditional no operands
6246 // Eg. CSINC x0, zr, zr, <cond>
6247 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6248 %{
6249 single_instruction;
6250 cr : EX1(read);
6251 dst : EX2(write);
6252 INS01 : ISS;
6253 ALU : EX2;
6254 %}
6255
6256 // Conditional 2 operand
6257 // EG. CSEL X0, X1, X2, <cond>
6258 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6259 %{
6260 single_instruction;
6261 cr : EX1(read);
6262 src1 : EX1(read);
6263 src2 : EX1(read);
6264 dst : EX2(write);
6265 INS01 : ISS;
6266 ALU : EX2;
6267 %}
6268
6269 // Conditional 2 operand
6270 // EG. CSEL X0, X1, X2, <cond>
6271 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6272 %{
6273 single_instruction;
6274 cr : EX1(read);
6275 src : EX1(read);
6276 dst : EX2(write);
6277 INS01 : ISS;
6278 ALU : EX2;
6279 %}
6280
6281 //------- Multiply pipeline operations --------------------
6282
6283 // Multiply reg-reg
6284 // Eg. MUL w0, w1, w2
6285 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6286 %{
6287 single_instruction;
6288 dst : WR(write);
6289 src1 : ISS(read);
6290 src2 : ISS(read);
6291 INS01 : ISS;
6292 MAC : WR;
6293 %}
6294
6295 // Multiply accumulate
6296 // Eg. MADD w0, w1, w2, w3
6297 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6298 %{
6299 single_instruction;
6300 dst : WR(write);
6301 src1 : ISS(read);
6302 src2 : ISS(read);
6303 src3 : ISS(read);
6304 INS01 : ISS;
6305 MAC : WR;
6306 %}
6307
6308 // Eg. MUL w0, w1, w2
6309 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6310 %{
6311 single_instruction;
6312 fixed_latency(3); // Maximum latency for 64 bit mul
6313 dst : WR(write);
6314 src1 : ISS(read);
6315 src2 : ISS(read);
6316 INS01 : ISS;
6317 MAC : WR;
6318 %}
6319
6320 // Multiply accumulate
6321 // Eg. MADD w0, w1, w2, w3
6322 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6323 %{
6324 single_instruction;
6325 fixed_latency(3); // Maximum latency for 64 bit mul
6326 dst : WR(write);
6327 src1 : ISS(read);
6328 src2 : ISS(read);
6329 src3 : ISS(read);
6330 INS01 : ISS;
6331 MAC : WR;
6332 %}
6333
6334 //------- Divide pipeline operations --------------------
6335
6336 // Eg. SDIV w0, w1, w2
6337 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6338 %{
6339 single_instruction;
6340 fixed_latency(8); // Maximum latency for 32 bit divide
6341 dst : WR(write);
6342 src1 : ISS(read);
6343 src2 : ISS(read);
6344 INS0 : ISS; // Can only dual issue as instruction 0
6345 DIV : WR;
6346 %}
6347
6348 // Eg. SDIV x0, x1, x2
6349 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6350 %{
6351 single_instruction;
6352 fixed_latency(16); // Maximum latency for 64 bit divide
6353 dst : WR(write);
6354 src1 : ISS(read);
6355 src2 : ISS(read);
6356 INS0 : ISS; // Can only dual issue as instruction 0
6357 DIV : WR;
6358 %}
6359
6360 //------- Load pipeline operations ------------------------
6361
6362 // Load - prefetch
6363 // Eg. PFRM <mem>
6364 pipe_class iload_prefetch(memory mem)
6365 %{
6366 single_instruction;
6367 mem : ISS(read);
6368 INS01 : ISS;
6369 LDST : WR;
6370 %}
6371
6372 // Load - reg, mem
6373 // Eg. LDR x0, <mem>
6374 pipe_class iload_reg_mem(iRegI dst, memory mem)
6375 %{
6376 single_instruction;
6377 dst : WR(write);
6378 mem : ISS(read);
6379 INS01 : ISS;
6380 LDST : WR;
6381 %}
6382
6383 // Load - reg, reg
6384 // Eg. LDR x0, [sp, x1]
6385 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6386 %{
6387 single_instruction;
6388 dst : WR(write);
6389 src : ISS(read);
6390 INS01 : ISS;
6391 LDST : WR;
6392 %}
6393
6394 //------- Store pipeline operations -----------------------
6395
6396 // Store - zr, mem
6397 // Eg. STR zr, <mem>
6398 pipe_class istore_mem(memory mem)
6399 %{
6400 single_instruction;
6401 mem : ISS(read);
6402 INS01 : ISS;
6403 LDST : WR;
6404 %}
6405
6406 // Store - reg, mem
6407 // Eg. STR x0, <mem>
6408 pipe_class istore_reg_mem(iRegI src, memory mem)
6409 %{
6410 single_instruction;
6411 mem : ISS(read);
6412 src : EX2(read);
6413 INS01 : ISS;
6414 LDST : WR;
6415 %}
6416
6417 // Store - reg, reg
6418 // Eg. STR x0, [sp, x1]
6419 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6420 %{
6421 single_instruction;
6422 dst : ISS(read);
6423 src : EX2(read);
6424 INS01 : ISS;
6425 LDST : WR;
6426 %}
6427
6428 //------- Store pipeline operations -----------------------
6429
6430 // Branch
6431 pipe_class pipe_branch()
6432 %{
6433 single_instruction;
6434 INS01 : ISS;
6435 BRANCH : EX1;
6436 %}
6437
6438 // Conditional branch
6439 pipe_class pipe_branch_cond(rFlagsReg cr)
6440 %{
6441 single_instruction;
6442 cr : EX1(read);
6443 INS01 : ISS;
6444 BRANCH : EX1;
6445 %}
6446
6447 // Compare & Branch
6448 // EG. CBZ/CBNZ
6449 pipe_class pipe_cmp_branch(iRegI op1)
6450 %{
6451 single_instruction;
6452 op1 : EX1(read);
6453 INS01 : ISS;
6454 BRANCH : EX1;
6455 %}
6456
6457 //------- Synchronisation operations ----------------------
6458
6459 // Any operation requiring serialization.
6460 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6461 pipe_class pipe_serial()
6462 %{
6463 single_instruction;
6464 force_serialization;
6465 fixed_latency(16);
6466 INS01 : ISS(2); // Cannot dual issue with any other instruction
6467 LDST : WR;
6468 %}
6469
6470 // Generic big/slow expanded idiom - also serialized
6471 pipe_class pipe_slow()
6472 %{
6473 instruction_count(10);
6474 multiple_bundles;
6475 force_serialization;
6476 fixed_latency(16);
6477 INS01 : ISS(2); // Cannot dual issue with any other instruction
6478 LDST : WR;
6479 %}
6480
6481 // Empty pipeline class
6482 pipe_class pipe_class_empty()
6483 %{
6484 single_instruction;
6485 fixed_latency(0);
6486 %}
6487
6488 // Default pipeline class.
6489 pipe_class pipe_class_default()
6490 %{
6491 single_instruction;
6492 fixed_latency(2);
6493 %}
6494
6495 // Pipeline class for compares.
6496 pipe_class pipe_class_compare()
6497 %{
6498 single_instruction;
6499 fixed_latency(16);
6500 %}
6501
6502 // Pipeline class for memory operations.
6503 pipe_class pipe_class_memory()
6504 %{
6505 single_instruction;
6506 fixed_latency(16);
6507 %}
6508
6509 // Pipeline class for call.
6510 pipe_class pipe_class_call()
6511 %{
6512 single_instruction;
6513 fixed_latency(100);
6514 %}
6515
6516 // Define the class for the Nop node.
6517 define %{
6518 MachNop = pipe_class_empty;
6519 %}
6520
6521 %}
6522 //----------INSTRUCTIONS-------------------------------------------------------
6523 //
6524 // match -- States which machine-independent subtree may be replaced
6525 // by this instruction.
6526 // ins_cost -- The estimated cost of this instruction is used by instruction
6527 // selection to identify a minimum cost tree of machine
6528 // instructions that matches a tree of machine-independent
6529 // instructions.
6530 // format -- A string providing the disassembly for this instruction.
6531 // The value of an instruction's operand may be inserted
6532 // by referring to it with a '$' prefix.
6533 // opcode -- Three instruction opcodes may be provided. These are referred
6534 // to within an encode class as $primary, $secondary, and $tertiary
6535 // rrspectively. The primary opcode is commonly used to
6536 // indicate the type of machine instruction, while secondary
6537 // and tertiary are often used for prefix options or addressing
6538 // modes.
6539 // ins_encode -- A list of encode classes with parameters. The encode class
6540 // name must have been defined in an 'enc_class' specification
6541 // in the encode section of the architecture description.
6542
6543 // ============================================================================
6544 // Memory (Load/Store) Instructions
6545
6546 // Load Instructions
6547
6548 // Load Byte (8 bit signed)
6549 instruct loadB(iRegINoSp dst, memory1 mem)
6550 %{
6551 match(Set dst (LoadB mem));
6552 predicate(!needs_acquiring_load(n));
6553
6554 ins_cost(4 * INSN_COST);
6555 format %{ "ldrsbw $dst, $mem\t# byte" %}
6556
6557 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6558
6559 ins_pipe(iload_reg_mem);
6560 %}
6561
6562 // Load Byte (8 bit signed) into long
6563 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6564 %{
6565 match(Set dst (ConvI2L (LoadB mem)));
6566 predicate(!needs_acquiring_load(n->in(1)));
6567
6568 ins_cost(4 * INSN_COST);
6569 format %{ "ldrsb $dst, $mem\t# byte" %}
6570
6571 ins_encode(aarch64_enc_ldrsb(dst, mem));
6572
6573 ins_pipe(iload_reg_mem);
6574 %}
6575
6576 // Load Byte (8 bit unsigned)
6577 instruct loadUB(iRegINoSp dst, memory1 mem)
6578 %{
6579 match(Set dst (LoadUB mem));
6580 predicate(!needs_acquiring_load(n));
6581
6582 ins_cost(4 * INSN_COST);
6583 format %{ "ldrbw $dst, $mem\t# byte" %}
6584
6585 ins_encode(aarch64_enc_ldrb(dst, mem));
6586
6587 ins_pipe(iload_reg_mem);
6588 %}
6589
6590 // Load Byte (8 bit unsigned) into long
6591 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6592 %{
6593 match(Set dst (ConvI2L (LoadUB mem)));
6594 predicate(!needs_acquiring_load(n->in(1)));
6595
6596 ins_cost(4 * INSN_COST);
6597 format %{ "ldrb $dst, $mem\t# byte" %}
6598
6599 ins_encode(aarch64_enc_ldrb(dst, mem));
6600
6601 ins_pipe(iload_reg_mem);
6602 %}
6603
6604 // Load Short (16 bit signed)
6605 instruct loadS(iRegINoSp dst, memory2 mem)
6606 %{
6607 match(Set dst (LoadS mem));
6608 predicate(!needs_acquiring_load(n));
6609
6610 ins_cost(4 * INSN_COST);
6611 format %{ "ldrshw $dst, $mem\t# short" %}
6612
6613 ins_encode(aarch64_enc_ldrshw(dst, mem));
6614
6615 ins_pipe(iload_reg_mem);
6616 %}
6617
6618 // Load Short (16 bit signed) into long
6619 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6620 %{
6621 match(Set dst (ConvI2L (LoadS mem)));
6622 predicate(!needs_acquiring_load(n->in(1)));
6623
6624 ins_cost(4 * INSN_COST);
6625 format %{ "ldrsh $dst, $mem\t# short" %}
6626
6627 ins_encode(aarch64_enc_ldrsh(dst, mem));
6628
6629 ins_pipe(iload_reg_mem);
6630 %}
6631
6632 // Load Char (16 bit unsigned)
6633 instruct loadUS(iRegINoSp dst, memory2 mem)
6634 %{
6635 match(Set dst (LoadUS mem));
6636 predicate(!needs_acquiring_load(n));
6637
6638 ins_cost(4 * INSN_COST);
6639 format %{ "ldrh $dst, $mem\t# short" %}
6640
6641 ins_encode(aarch64_enc_ldrh(dst, mem));
6642
6643 ins_pipe(iload_reg_mem);
6644 %}
6645
6646 // Load Short/Char (16 bit unsigned) into long
6647 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6648 %{
6649 match(Set dst (ConvI2L (LoadUS mem)));
6650 predicate(!needs_acquiring_load(n->in(1)));
6651
6652 ins_cost(4 * INSN_COST);
6653 format %{ "ldrh $dst, $mem\t# short" %}
6654
6655 ins_encode(aarch64_enc_ldrh(dst, mem));
6656
6657 ins_pipe(iload_reg_mem);
6658 %}
6659
6660 // Load Integer (32 bit signed)
6661 instruct loadI(iRegINoSp dst, memory4 mem)
6662 %{
6663 match(Set dst (LoadI mem));
6664 predicate(!needs_acquiring_load(n));
6665
6666 ins_cost(4 * INSN_COST);
6667 format %{ "ldrw $dst, $mem\t# int" %}
6668
6669 ins_encode(aarch64_enc_ldrw(dst, mem));
6670
6671 ins_pipe(iload_reg_mem);
6672 %}
6673
6674 // Load Integer (32 bit signed) into long
6675 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6676 %{
6677 match(Set dst (ConvI2L (LoadI mem)));
6678 predicate(!needs_acquiring_load(n->in(1)));
6679
6680 ins_cost(4 * INSN_COST);
6681 format %{ "ldrsw $dst, $mem\t# int" %}
6682
6683 ins_encode(aarch64_enc_ldrsw(dst, mem));
6684
6685 ins_pipe(iload_reg_mem);
6686 %}
6687
6688 // Load Integer (32 bit unsigned) into long
6689 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6690 %{
6691 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6692 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6693
6694 ins_cost(4 * INSN_COST);
6695 format %{ "ldrw $dst, $mem\t# int" %}
6696
6697 ins_encode(aarch64_enc_ldrw(dst, mem));
6698
6699 ins_pipe(iload_reg_mem);
6700 %}
6701
6702 // Load Long (64 bit signed)
6703 instruct loadL(iRegLNoSp dst, memory8 mem)
6704 %{
6705 match(Set dst (LoadL mem));
6706 predicate(!needs_acquiring_load(n));
6707
6708 ins_cost(4 * INSN_COST);
6709 format %{ "ldr $dst, $mem\t# int" %}
6710
6711 ins_encode(aarch64_enc_ldr(dst, mem));
6712
6713 ins_pipe(iload_reg_mem);
6714 %}
6715
6716 // Load Range
6717 instruct loadRange(iRegINoSp dst, memory4 mem)
6718 %{
6719 match(Set dst (LoadRange mem));
6720
6721 ins_cost(4 * INSN_COST);
6722 format %{ "ldrw $dst, $mem\t# range" %}
6723
6724 ins_encode(aarch64_enc_ldrw(dst, mem));
6725
6726 ins_pipe(iload_reg_mem);
6727 %}
6728
6729 // Load Pointer
6730 instruct loadP(iRegPNoSp dst, memory8 mem)
6731 %{
6732 match(Set dst (LoadP mem));
6733 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6734
6735 ins_cost(4 * INSN_COST);
6736 format %{ "ldr $dst, $mem\t# ptr" %}
6737
6738 ins_encode(aarch64_enc_ldr(dst, mem));
6739
6740 ins_pipe(iload_reg_mem);
6741 %}
6742
6743 // Load Compressed Pointer
6744 instruct loadN(iRegNNoSp dst, memory4 mem)
6745 %{
6746 match(Set dst (LoadN mem));
6747 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6748
6749 ins_cost(4 * INSN_COST);
6750 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6751
6752 ins_encode(aarch64_enc_ldrw(dst, mem));
6753
6754 ins_pipe(iload_reg_mem);
6755 %}
6756
6757 // Load Klass Pointer
6758 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6759 %{
6760 match(Set dst (LoadKlass mem));
6761 predicate(!needs_acquiring_load(n));
6762
6763 ins_cost(4 * INSN_COST);
6764 format %{ "ldr $dst, $mem\t# class" %}
6765
6766 ins_encode(aarch64_enc_ldr(dst, mem));
6767
6768 ins_pipe(iload_reg_mem);
6769 %}
6770
6771 // Load Narrow Klass Pointer
6772 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6773 %{
6774 match(Set dst (LoadNKlass mem));
6775 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6776
6777 ins_cost(4 * INSN_COST);
6778 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6779
6780 ins_encode(aarch64_enc_ldrw(dst, mem));
6781
6782 ins_pipe(iload_reg_mem);
6783 %}
6784
6785 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6786 %{
6787 match(Set dst (LoadNKlass mem));
6788 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6789
6790 ins_cost(4 * INSN_COST);
6791 format %{
6792 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6793 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6794 %}
6795 ins_encode %{
6796 // inlined aarch64_enc_ldrw
6797 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6798 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6799 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6800 %}
6801 ins_pipe(iload_reg_mem);
6802 %}
6803
6804 // Load Float
6805 instruct loadF(vRegF dst, memory4 mem)
6806 %{
6807 match(Set dst (LoadF mem));
6808 predicate(!needs_acquiring_load(n));
6809
6810 ins_cost(4 * INSN_COST);
6811 format %{ "ldrs $dst, $mem\t# float" %}
6812
6813 ins_encode( aarch64_enc_ldrs(dst, mem) );
6814
6815 ins_pipe(pipe_class_memory);
6816 %}
6817
6818 // Load Double
6819 instruct loadD(vRegD dst, memory8 mem)
6820 %{
6821 match(Set dst (LoadD mem));
6822 predicate(!needs_acquiring_load(n));
6823
6824 ins_cost(4 * INSN_COST);
6825 format %{ "ldrd $dst, $mem\t# double" %}
6826
6827 ins_encode( aarch64_enc_ldrd(dst, mem) );
6828
6829 ins_pipe(pipe_class_memory);
6830 %}
6831
6832
6833 // Load Int Constant
6834 instruct loadConI(iRegINoSp dst, immI src)
6835 %{
6836 match(Set dst src);
6837
6838 ins_cost(INSN_COST);
6839 format %{ "mov $dst, $src\t# int" %}
6840
6841 ins_encode( aarch64_enc_movw_imm(dst, src) );
6842
6843 ins_pipe(ialu_imm);
6844 %}
6845
6846 // Load Long Constant
6847 instruct loadConL(iRegLNoSp dst, immL src)
6848 %{
6849 match(Set dst src);
6850
6851 ins_cost(INSN_COST);
6852 format %{ "mov $dst, $src\t# long" %}
6853
6854 ins_encode( aarch64_enc_mov_imm(dst, src) );
6855
6856 ins_pipe(ialu_imm);
6857 %}
6858
6859 // Load Pointer Constant
6860
6861 instruct loadConP(iRegPNoSp dst, immP con)
6862 %{
6863 match(Set dst con);
6864
6865 ins_cost(INSN_COST * 4);
6866 format %{
6867 "mov $dst, $con\t# ptr\n\t"
6868 %}
6869
6870 ins_encode(aarch64_enc_mov_p(dst, con));
6871
6872 ins_pipe(ialu_imm);
6873 %}
6874
6875 // Load Null Pointer Constant
6876
6877 instruct loadConP0(iRegPNoSp dst, immP0 con)
6878 %{
6879 match(Set dst con);
6880
6881 ins_cost(INSN_COST);
6882 format %{ "mov $dst, $con\t# nullptr ptr" %}
6883
6884 ins_encode(aarch64_enc_mov_p0(dst, con));
6885
6886 ins_pipe(ialu_imm);
6887 %}
6888
6889 // Load Pointer Constant One
6890
6891 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6892 %{
6893 match(Set dst con);
6894
6895 ins_cost(INSN_COST);
6896 format %{ "mov $dst, $con\t# nullptr ptr" %}
6897
6898 ins_encode(aarch64_enc_mov_p1(dst, con));
6899
6900 ins_pipe(ialu_imm);
6901 %}
6902
6903 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6904 %{
6905 match(Set dst con);
6906
6907 ins_cost(INSN_COST);
6908 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6909
6910 ins_encode %{
6911 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6912 %}
6913
6914 ins_pipe(ialu_imm);
6915 %}
6916
6917 // Load Narrow Pointer Constant
6918
6919 instruct loadConN(iRegNNoSp dst, immN con)
6920 %{
6921 match(Set dst con);
6922
6923 ins_cost(INSN_COST * 4);
6924 format %{ "mov $dst, $con\t# compressed ptr" %}
6925
6926 ins_encode(aarch64_enc_mov_n(dst, con));
6927
6928 ins_pipe(ialu_imm);
6929 %}
6930
6931 // Load Narrow Null Pointer Constant
6932
6933 instruct loadConN0(iRegNNoSp dst, immN0 con)
6934 %{
6935 match(Set dst con);
6936
6937 ins_cost(INSN_COST);
6938 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6939
6940 ins_encode(aarch64_enc_mov_n0(dst, con));
6941
6942 ins_pipe(ialu_imm);
6943 %}
6944
6945 // Load Narrow Klass Constant
6946
6947 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6948 %{
6949 match(Set dst con);
6950
6951 ins_cost(INSN_COST);
6952 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6953
6954 ins_encode(aarch64_enc_mov_nk(dst, con));
6955
6956 ins_pipe(ialu_imm);
6957 %}
6958
6959 // Load Packed Float Constant
6960
6961 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6962 match(Set dst con);
6963 ins_cost(INSN_COST * 4);
6964 format %{ "fmovs $dst, $con"%}
6965 ins_encode %{
6966 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6967 %}
6968
6969 ins_pipe(fp_imm_s);
6970 %}
6971
6972 // Load Float Constant
6973
6974 instruct loadConF(vRegF dst, immF con) %{
6975 match(Set dst con);
6976
6977 ins_cost(INSN_COST * 4);
6978
6979 format %{
6980 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6981 %}
6982
6983 ins_encode %{
6984 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6985 %}
6986
6987 ins_pipe(fp_load_constant_s);
6988 %}
6989
6990 // Load Packed Double Constant
6991
6992 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6993 match(Set dst con);
6994 ins_cost(INSN_COST);
6995 format %{ "fmovd $dst, $con"%}
6996 ins_encode %{
6997 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6998 %}
6999
7000 ins_pipe(fp_imm_d);
7001 %}
7002
7003 // Load Double Constant
7004
7005 instruct loadConD(vRegD dst, immD con) %{
7006 match(Set dst con);
7007
7008 ins_cost(INSN_COST * 5);
7009 format %{
7010 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7011 %}
7012
7013 ins_encode %{
7014 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7015 %}
7016
7017 ins_pipe(fp_load_constant_d);
7018 %}
7019
7020 // Load Half Float Constant
7021 instruct loadConH(vRegF dst, immH con) %{
7022 match(Set dst con);
7023 format %{ "mov rscratch1, $con\n\t"
7024 "fmov $dst, rscratch1"
7025 %}
7026 ins_encode %{
7027 __ movw(rscratch1, (uint32_t)$con$$constant);
7028 __ fmovs($dst$$FloatRegister, rscratch1);
7029 %}
7030 ins_pipe(pipe_class_default);
7031 %}
7032
7033 // Store Instructions
7034
7035 // Store Byte
7036 instruct storeB(iRegIorL2I src, memory1 mem)
7037 %{
7038 match(Set mem (StoreB mem src));
7039 predicate(!needs_releasing_store(n));
7040
7041 ins_cost(INSN_COST);
7042 format %{ "strb $src, $mem\t# byte" %}
7043
7044 ins_encode(aarch64_enc_strb(src, mem));
7045
7046 ins_pipe(istore_reg_mem);
7047 %}
7048
7049
7050 instruct storeimmB0(immI0 zero, memory1 mem)
7051 %{
7052 match(Set mem (StoreB mem zero));
7053 predicate(!needs_releasing_store(n));
7054
7055 ins_cost(INSN_COST);
7056 format %{ "strb rscractch2, $mem\t# byte" %}
7057
7058 ins_encode(aarch64_enc_strb0(mem));
7059
7060 ins_pipe(istore_mem);
7061 %}
7062
7063 // Store Char/Short
7064 instruct storeC(iRegIorL2I src, memory2 mem)
7065 %{
7066 match(Set mem (StoreC mem src));
7067 predicate(!needs_releasing_store(n));
7068
7069 ins_cost(INSN_COST);
7070 format %{ "strh $src, $mem\t# short" %}
7071
7072 ins_encode(aarch64_enc_strh(src, mem));
7073
7074 ins_pipe(istore_reg_mem);
7075 %}
7076
7077 instruct storeimmC0(immI0 zero, memory2 mem)
7078 %{
7079 match(Set mem (StoreC mem zero));
7080 predicate(!needs_releasing_store(n));
7081
7082 ins_cost(INSN_COST);
7083 format %{ "strh zr, $mem\t# short" %}
7084
7085 ins_encode(aarch64_enc_strh0(mem));
7086
7087 ins_pipe(istore_mem);
7088 %}
7089
7090 // Store Integer
7091
7092 instruct storeI(iRegIorL2I src, memory4 mem)
7093 %{
7094 match(Set mem(StoreI mem src));
7095 predicate(!needs_releasing_store(n));
7096
7097 ins_cost(INSN_COST);
7098 format %{ "strw $src, $mem\t# int" %}
7099
7100 ins_encode(aarch64_enc_strw(src, mem));
7101
7102 ins_pipe(istore_reg_mem);
7103 %}
7104
7105 instruct storeimmI0(immI0 zero, memory4 mem)
7106 %{
7107 match(Set mem(StoreI mem zero));
7108 predicate(!needs_releasing_store(n));
7109
7110 ins_cost(INSN_COST);
7111 format %{ "strw zr, $mem\t# int" %}
7112
7113 ins_encode(aarch64_enc_strw0(mem));
7114
7115 ins_pipe(istore_mem);
7116 %}
7117
7118 // Store Long (64 bit signed)
7119 instruct storeL(iRegL src, memory8 mem)
7120 %{
7121 match(Set mem (StoreL mem src));
7122 predicate(!needs_releasing_store(n));
7123
7124 ins_cost(INSN_COST);
7125 format %{ "str $src, $mem\t# int" %}
7126
7127 ins_encode(aarch64_enc_str(src, mem));
7128
7129 ins_pipe(istore_reg_mem);
7130 %}
7131
7132 // Store Long (64 bit signed)
7133 instruct storeimmL0(immL0 zero, memory8 mem)
7134 %{
7135 match(Set mem (StoreL mem zero));
7136 predicate(!needs_releasing_store(n));
7137
7138 ins_cost(INSN_COST);
7139 format %{ "str zr, $mem\t# int" %}
7140
7141 ins_encode(aarch64_enc_str0(mem));
7142
7143 ins_pipe(istore_mem);
7144 %}
7145
7146 // Store Pointer
7147 instruct storeP(iRegP src, memory8 mem)
7148 %{
7149 match(Set mem (StoreP mem src));
7150 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7151
7152 ins_cost(INSN_COST);
7153 format %{ "str $src, $mem\t# ptr" %}
7154
7155 ins_encode(aarch64_enc_str(src, mem));
7156
7157 ins_pipe(istore_reg_mem);
7158 %}
7159
7160 // Store Pointer
7161 instruct storeimmP0(immP0 zero, memory8 mem)
7162 %{
7163 match(Set mem (StoreP mem zero));
7164 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7165
7166 ins_cost(INSN_COST);
7167 format %{ "str zr, $mem\t# ptr" %}
7168
7169 ins_encode(aarch64_enc_str0(mem));
7170
7171 ins_pipe(istore_mem);
7172 %}
7173
7174 // Store Compressed Pointer
7175 instruct storeN(iRegN src, memory4 mem)
7176 %{
7177 match(Set mem (StoreN mem src));
7178 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7179
7180 ins_cost(INSN_COST);
7181 format %{ "strw $src, $mem\t# compressed ptr" %}
7182
7183 ins_encode(aarch64_enc_strw(src, mem));
7184
7185 ins_pipe(istore_reg_mem);
7186 %}
7187
7188 instruct storeImmN0(immN0 zero, memory4 mem)
7189 %{
7190 match(Set mem (StoreN mem zero));
7191 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7192
7193 ins_cost(INSN_COST);
7194 format %{ "strw zr, $mem\t# compressed ptr" %}
7195
7196 ins_encode(aarch64_enc_strw0(mem));
7197
7198 ins_pipe(istore_mem);
7199 %}
7200
7201 // Store Float
7202 instruct storeF(vRegF src, memory4 mem)
7203 %{
7204 match(Set mem (StoreF mem src));
7205 predicate(!needs_releasing_store(n));
7206
7207 ins_cost(INSN_COST);
7208 format %{ "strs $src, $mem\t# float" %}
7209
7210 ins_encode( aarch64_enc_strs(src, mem) );
7211
7212 ins_pipe(pipe_class_memory);
7213 %}
7214
7215 // TODO
7216 // implement storeImmF0 and storeFImmPacked
7217
7218 // Store Double
7219 instruct storeD(vRegD src, memory8 mem)
7220 %{
7221 match(Set mem (StoreD mem src));
7222 predicate(!needs_releasing_store(n));
7223
7224 ins_cost(INSN_COST);
7225 format %{ "strd $src, $mem\t# double" %}
7226
7227 ins_encode( aarch64_enc_strd(src, mem) );
7228
7229 ins_pipe(pipe_class_memory);
7230 %}
7231
7232 // Store Compressed Klass Pointer
7233 instruct storeNKlass(iRegN src, memory4 mem)
7234 %{
7235 predicate(!needs_releasing_store(n));
7236 match(Set mem (StoreNKlass mem src));
7237
7238 ins_cost(INSN_COST);
7239 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7240
7241 ins_encode(aarch64_enc_strw(src, mem));
7242
7243 ins_pipe(istore_reg_mem);
7244 %}
7245
7246 // TODO
7247 // implement storeImmD0 and storeDImmPacked
7248
7249 // prefetch instructions
7250 // Must be safe to execute with invalid address (cannot fault).
7251
7252 instruct prefetchalloc( memory8 mem ) %{
7253 match(PrefetchAllocation mem);
7254
7255 ins_cost(INSN_COST);
7256 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7257
7258 ins_encode( aarch64_enc_prefetchw(mem) );
7259
7260 ins_pipe(iload_prefetch);
7261 %}
7262
7263 // ---------------- volatile loads and stores ----------------
7264
7265 // Load Byte (8 bit signed)
7266 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7267 %{
7268 match(Set dst (LoadB mem));
7269
7270 ins_cost(VOLATILE_REF_COST);
7271 format %{ "ldarsb $dst, $mem\t# byte" %}
7272
7273 ins_encode(aarch64_enc_ldarsb(dst, mem));
7274
7275 ins_pipe(pipe_serial);
7276 %}
7277
7278 // Load Byte (8 bit signed) into long
7279 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7280 %{
7281 match(Set dst (ConvI2L (LoadB mem)));
7282
7283 ins_cost(VOLATILE_REF_COST);
7284 format %{ "ldarsb $dst, $mem\t# byte" %}
7285
7286 ins_encode(aarch64_enc_ldarsb(dst, mem));
7287
7288 ins_pipe(pipe_serial);
7289 %}
7290
7291 // Load Byte (8 bit unsigned)
7292 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7293 %{
7294 match(Set dst (LoadUB mem));
7295
7296 ins_cost(VOLATILE_REF_COST);
7297 format %{ "ldarb $dst, $mem\t# byte" %}
7298
7299 ins_encode(aarch64_enc_ldarb(dst, mem));
7300
7301 ins_pipe(pipe_serial);
7302 %}
7303
7304 // Load Byte (8 bit unsigned) into long
7305 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7306 %{
7307 match(Set dst (ConvI2L (LoadUB mem)));
7308
7309 ins_cost(VOLATILE_REF_COST);
7310 format %{ "ldarb $dst, $mem\t# byte" %}
7311
7312 ins_encode(aarch64_enc_ldarb(dst, mem));
7313
7314 ins_pipe(pipe_serial);
7315 %}
7316
7317 // Load Short (16 bit signed)
7318 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7319 %{
7320 match(Set dst (LoadS mem));
7321
7322 ins_cost(VOLATILE_REF_COST);
7323 format %{ "ldarshw $dst, $mem\t# short" %}
7324
7325 ins_encode(aarch64_enc_ldarshw(dst, mem));
7326
7327 ins_pipe(pipe_serial);
7328 %}
7329
7330 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7331 %{
7332 match(Set dst (LoadUS mem));
7333
7334 ins_cost(VOLATILE_REF_COST);
7335 format %{ "ldarhw $dst, $mem\t# short" %}
7336
7337 ins_encode(aarch64_enc_ldarhw(dst, mem));
7338
7339 ins_pipe(pipe_serial);
7340 %}
7341
7342 // Load Short/Char (16 bit unsigned) into long
7343 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7344 %{
7345 match(Set dst (ConvI2L (LoadUS mem)));
7346
7347 ins_cost(VOLATILE_REF_COST);
7348 format %{ "ldarh $dst, $mem\t# short" %}
7349
7350 ins_encode(aarch64_enc_ldarh(dst, mem));
7351
7352 ins_pipe(pipe_serial);
7353 %}
7354
7355 // Load Short/Char (16 bit signed) into long
7356 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7357 %{
7358 match(Set dst (ConvI2L (LoadS mem)));
7359
7360 ins_cost(VOLATILE_REF_COST);
7361 format %{ "ldarh $dst, $mem\t# short" %}
7362
7363 ins_encode(aarch64_enc_ldarsh(dst, mem));
7364
7365 ins_pipe(pipe_serial);
7366 %}
7367
7368 // Load Integer (32 bit signed)
7369 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7370 %{
7371 match(Set dst (LoadI mem));
7372
7373 ins_cost(VOLATILE_REF_COST);
7374 format %{ "ldarw $dst, $mem\t# int" %}
7375
7376 ins_encode(aarch64_enc_ldarw(dst, mem));
7377
7378 ins_pipe(pipe_serial);
7379 %}
7380
7381 // Load Integer (32 bit unsigned) into long
7382 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7383 %{
7384 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7385
7386 ins_cost(VOLATILE_REF_COST);
7387 format %{ "ldarw $dst, $mem\t# int" %}
7388
7389 ins_encode(aarch64_enc_ldarw(dst, mem));
7390
7391 ins_pipe(pipe_serial);
7392 %}
7393
7394 // Load Long (64 bit signed)
7395 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7396 %{
7397 match(Set dst (LoadL mem));
7398
7399 ins_cost(VOLATILE_REF_COST);
7400 format %{ "ldar $dst, $mem\t# int" %}
7401
7402 ins_encode(aarch64_enc_ldar(dst, mem));
7403
7404 ins_pipe(pipe_serial);
7405 %}
7406
7407 // Load Pointer
7408 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7409 %{
7410 match(Set dst (LoadP mem));
7411 predicate(n->as_Load()->barrier_data() == 0);
7412
7413 ins_cost(VOLATILE_REF_COST);
7414 format %{ "ldar $dst, $mem\t# ptr" %}
7415
7416 ins_encode(aarch64_enc_ldar(dst, mem));
7417
7418 ins_pipe(pipe_serial);
7419 %}
7420
7421 // Load Compressed Pointer
7422 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7423 %{
7424 match(Set dst (LoadN mem));
7425 predicate(n->as_Load()->barrier_data() == 0);
7426
7427 ins_cost(VOLATILE_REF_COST);
7428 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7429
7430 ins_encode(aarch64_enc_ldarw(dst, mem));
7431
7432 ins_pipe(pipe_serial);
7433 %}
7434
7435 // Load Float
7436 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7437 %{
7438 match(Set dst (LoadF mem));
7439
7440 ins_cost(VOLATILE_REF_COST);
7441 format %{ "ldars $dst, $mem\t# float" %}
7442
7443 ins_encode( aarch64_enc_fldars(dst, mem) );
7444
7445 ins_pipe(pipe_serial);
7446 %}
7447
7448 // Load Double
7449 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7450 %{
7451 match(Set dst (LoadD mem));
7452
7453 ins_cost(VOLATILE_REF_COST);
7454 format %{ "ldard $dst, $mem\t# double" %}
7455
7456 ins_encode( aarch64_enc_fldard(dst, mem) );
7457
7458 ins_pipe(pipe_serial);
7459 %}
7460
7461 // Store Byte
7462 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7463 %{
7464 match(Set mem (StoreB mem src));
7465
7466 ins_cost(VOLATILE_REF_COST);
7467 format %{ "stlrb $src, $mem\t# byte" %}
7468
7469 ins_encode(aarch64_enc_stlrb(src, mem));
7470
7471 ins_pipe(pipe_class_memory);
7472 %}
7473
7474 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7475 %{
7476 match(Set mem (StoreB mem zero));
7477
7478 ins_cost(VOLATILE_REF_COST);
7479 format %{ "stlrb zr, $mem\t# byte" %}
7480
7481 ins_encode(aarch64_enc_stlrb0(mem));
7482
7483 ins_pipe(pipe_class_memory);
7484 %}
7485
7486 // Store Char/Short
7487 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7488 %{
7489 match(Set mem (StoreC mem src));
7490
7491 ins_cost(VOLATILE_REF_COST);
7492 format %{ "stlrh $src, $mem\t# short" %}
7493
7494 ins_encode(aarch64_enc_stlrh(src, mem));
7495
7496 ins_pipe(pipe_class_memory);
7497 %}
7498
7499 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7500 %{
7501 match(Set mem (StoreC mem zero));
7502
7503 ins_cost(VOLATILE_REF_COST);
7504 format %{ "stlrh zr, $mem\t# short" %}
7505
7506 ins_encode(aarch64_enc_stlrh0(mem));
7507
7508 ins_pipe(pipe_class_memory);
7509 %}
7510
7511 // Store Integer
7512
7513 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7514 %{
7515 match(Set mem(StoreI mem src));
7516
7517 ins_cost(VOLATILE_REF_COST);
7518 format %{ "stlrw $src, $mem\t# int" %}
7519
7520 ins_encode(aarch64_enc_stlrw(src, mem));
7521
7522 ins_pipe(pipe_class_memory);
7523 %}
7524
7525 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7526 %{
7527 match(Set mem(StoreI mem zero));
7528
7529 ins_cost(VOLATILE_REF_COST);
7530 format %{ "stlrw zr, $mem\t# int" %}
7531
7532 ins_encode(aarch64_enc_stlrw0(mem));
7533
7534 ins_pipe(pipe_class_memory);
7535 %}
7536
7537 // Store Long (64 bit signed)
7538 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7539 %{
7540 match(Set mem (StoreL mem src));
7541
7542 ins_cost(VOLATILE_REF_COST);
7543 format %{ "stlr $src, $mem\t# int" %}
7544
7545 ins_encode(aarch64_enc_stlr(src, mem));
7546
7547 ins_pipe(pipe_class_memory);
7548 %}
7549
7550 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7551 %{
7552 match(Set mem (StoreL mem zero));
7553
7554 ins_cost(VOLATILE_REF_COST);
7555 format %{ "stlr zr, $mem\t# int" %}
7556
7557 ins_encode(aarch64_enc_stlr0(mem));
7558
7559 ins_pipe(pipe_class_memory);
7560 %}
7561
7562 // Store Pointer
7563 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7564 %{
7565 match(Set mem (StoreP mem src));
7566 predicate(n->as_Store()->barrier_data() == 0);
7567
7568 ins_cost(VOLATILE_REF_COST);
7569 format %{ "stlr $src, $mem\t# ptr" %}
7570
7571 ins_encode(aarch64_enc_stlr(src, mem));
7572
7573 ins_pipe(pipe_class_memory);
7574 %}
7575
7576 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7577 %{
7578 match(Set mem (StoreP mem zero));
7579 predicate(n->as_Store()->barrier_data() == 0);
7580
7581 ins_cost(VOLATILE_REF_COST);
7582 format %{ "stlr zr, $mem\t# ptr" %}
7583
7584 ins_encode(aarch64_enc_stlr0(mem));
7585
7586 ins_pipe(pipe_class_memory);
7587 %}
7588
7589 // Store Compressed Pointer
7590 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7591 %{
7592 match(Set mem (StoreN mem src));
7593 predicate(n->as_Store()->barrier_data() == 0);
7594
7595 ins_cost(VOLATILE_REF_COST);
7596 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7597
7598 ins_encode(aarch64_enc_stlrw(src, mem));
7599
7600 ins_pipe(pipe_class_memory);
7601 %}
7602
7603 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7604 %{
7605 match(Set mem (StoreN mem zero));
7606 predicate(n->as_Store()->barrier_data() == 0);
7607
7608 ins_cost(VOLATILE_REF_COST);
7609 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7610
7611 ins_encode(aarch64_enc_stlrw0(mem));
7612
7613 ins_pipe(pipe_class_memory);
7614 %}
7615
7616 // Store Float
7617 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7618 %{
7619 match(Set mem (StoreF mem src));
7620
7621 ins_cost(VOLATILE_REF_COST);
7622 format %{ "stlrs $src, $mem\t# float" %}
7623
7624 ins_encode( aarch64_enc_fstlrs(src, mem) );
7625
7626 ins_pipe(pipe_class_memory);
7627 %}
7628
7629 // TODO
7630 // implement storeImmF0 and storeFImmPacked
7631
7632 // Store Double
7633 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7634 %{
7635 match(Set mem (StoreD mem src));
7636
7637 ins_cost(VOLATILE_REF_COST);
7638 format %{ "stlrd $src, $mem\t# double" %}
7639
7640 ins_encode( aarch64_enc_fstlrd(src, mem) );
7641
7642 ins_pipe(pipe_class_memory);
7643 %}
7644
7645 // ---------------- end of volatile loads and stores ----------------
7646
7647 instruct cacheWB(indirect addr)
7648 %{
7649 predicate(VM_Version::supports_data_cache_line_flush());
7650 match(CacheWB addr);
7651
7652 ins_cost(100);
7653 format %{"cache wb $addr" %}
7654 ins_encode %{
7655 assert($addr->index_position() < 0, "should be");
7656 assert($addr$$disp == 0, "should be");
7657 __ cache_wb(Address($addr$$base$$Register, 0));
7658 %}
7659 ins_pipe(pipe_slow); // XXX
7660 %}
7661
7662 instruct cacheWBPreSync()
7663 %{
7664 predicate(VM_Version::supports_data_cache_line_flush());
7665 match(CacheWBPreSync);
7666
7667 ins_cost(100);
7668 format %{"cache wb presync" %}
7669 ins_encode %{
7670 __ cache_wbsync(true);
7671 %}
7672 ins_pipe(pipe_slow); // XXX
7673 %}
7674
7675 instruct cacheWBPostSync()
7676 %{
7677 predicate(VM_Version::supports_data_cache_line_flush());
7678 match(CacheWBPostSync);
7679
7680 ins_cost(100);
7681 format %{"cache wb postsync" %}
7682 ins_encode %{
7683 __ cache_wbsync(false);
7684 %}
7685 ins_pipe(pipe_slow); // XXX
7686 %}
7687
7688 // ============================================================================
7689 // BSWAP Instructions
7690
7691 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7692 match(Set dst (ReverseBytesI src));
7693
7694 ins_cost(INSN_COST);
7695 format %{ "revw $dst, $src" %}
7696
7697 ins_encode %{
7698 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7699 %}
7700
7701 ins_pipe(ialu_reg);
7702 %}
7703
7704 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7705 match(Set dst (ReverseBytesL src));
7706
7707 ins_cost(INSN_COST);
7708 format %{ "rev $dst, $src" %}
7709
7710 ins_encode %{
7711 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7712 %}
7713
7714 ins_pipe(ialu_reg);
7715 %}
7716
7717 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7718 match(Set dst (ReverseBytesUS src));
7719
7720 ins_cost(INSN_COST);
7721 format %{ "rev16w $dst, $src" %}
7722
7723 ins_encode %{
7724 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7725 %}
7726
7727 ins_pipe(ialu_reg);
7728 %}
7729
7730 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7731 match(Set dst (ReverseBytesS src));
7732
7733 ins_cost(INSN_COST);
7734 format %{ "rev16w $dst, $src\n\t"
7735 "sbfmw $dst, $dst, #0, #15" %}
7736
7737 ins_encode %{
7738 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7739 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7740 %}
7741
7742 ins_pipe(ialu_reg);
7743 %}
7744
7745 // ============================================================================
7746 // Zero Count Instructions
7747
7748 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7749 match(Set dst (CountLeadingZerosI src));
7750
7751 ins_cost(INSN_COST);
7752 format %{ "clzw $dst, $src" %}
7753 ins_encode %{
7754 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7755 %}
7756
7757 ins_pipe(ialu_reg);
7758 %}
7759
7760 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7761 match(Set dst (CountLeadingZerosL src));
7762
7763 ins_cost(INSN_COST);
7764 format %{ "clz $dst, $src" %}
7765 ins_encode %{
7766 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7767 %}
7768
7769 ins_pipe(ialu_reg);
7770 %}
7771
7772 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7773 match(Set dst (CountTrailingZerosI src));
7774
7775 ins_cost(INSN_COST * 2);
7776 format %{ "rbitw $dst, $src\n\t"
7777 "clzw $dst, $dst" %}
7778 ins_encode %{
7779 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7780 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7781 %}
7782
7783 ins_pipe(ialu_reg);
7784 %}
7785
7786 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7787 match(Set dst (CountTrailingZerosL src));
7788
7789 ins_cost(INSN_COST * 2);
7790 format %{ "rbit $dst, $src\n\t"
7791 "clz $dst, $dst" %}
7792 ins_encode %{
7793 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7794 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7795 %}
7796
7797 ins_pipe(ialu_reg);
7798 %}
7799
7800 //---------- Population Count Instructions -------------------------------------
7801 //
7802
7803 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7804 match(Set dst (PopCountI src));
7805 effect(TEMP tmp);
7806 ins_cost(INSN_COST * 13);
7807
7808 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7809 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7810 "addv $tmp, $tmp\t# vector (8B)\n\t"
7811 "mov $dst, $tmp\t# vector (1D)" %}
7812 ins_encode %{
7813 __ fmovs($tmp$$FloatRegister, $src$$Register);
7814 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7815 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7816 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7817 %}
7818
7819 ins_pipe(pipe_class_default);
7820 %}
7821
7822 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7823 match(Set dst (PopCountI (LoadI mem)));
7824 effect(TEMP tmp);
7825 ins_cost(INSN_COST * 13);
7826
7827 format %{ "ldrs $tmp, $mem\n\t"
7828 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7829 "addv $tmp, $tmp\t# vector (8B)\n\t"
7830 "mov $dst, $tmp\t# vector (1D)" %}
7831 ins_encode %{
7832 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7833 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7834 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7835 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7836 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7837 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7838 %}
7839
7840 ins_pipe(pipe_class_default);
7841 %}
7842
7843 // Note: Long.bitCount(long) returns an int.
7844 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7845 match(Set dst (PopCountL src));
7846 effect(TEMP tmp);
7847 ins_cost(INSN_COST * 13);
7848
7849 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7850 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7851 "addv $tmp, $tmp\t# vector (8B)\n\t"
7852 "mov $dst, $tmp\t# vector (1D)" %}
7853 ins_encode %{
7854 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7855 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7856 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7857 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7858 %}
7859
7860 ins_pipe(pipe_class_default);
7861 %}
7862
7863 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7864 match(Set dst (PopCountL (LoadL mem)));
7865 effect(TEMP tmp);
7866 ins_cost(INSN_COST * 13);
7867
7868 format %{ "ldrd $tmp, $mem\n\t"
7869 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7870 "addv $tmp, $tmp\t# vector (8B)\n\t"
7871 "mov $dst, $tmp\t# vector (1D)" %}
7872 ins_encode %{
7873 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7874 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7875 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7876 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7877 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7878 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7879 %}
7880
7881 ins_pipe(pipe_class_default);
7882 %}
7883
7884 // ============================================================================
7885 // VerifyVectorAlignment Instruction
7886
7887 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7888 match(Set addr (VerifyVectorAlignment addr mask));
7889 effect(KILL cr);
7890 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7891 ins_encode %{
7892 Label Lskip;
7893 // check if masked bits of addr are zero
7894 __ tst($addr$$Register, $mask$$constant);
7895 __ br(Assembler::EQ, Lskip);
7896 __ stop("verify_vector_alignment found a misaligned vector memory access");
7897 __ bind(Lskip);
7898 %}
7899 ins_pipe(pipe_slow);
7900 %}
7901
7902 // ============================================================================
7903 // MemBar Instruction
7904
7905 instruct load_fence() %{
7906 match(LoadFence);
7907 ins_cost(VOLATILE_REF_COST);
7908
7909 format %{ "load_fence" %}
7910
7911 ins_encode %{
7912 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7913 %}
7914 ins_pipe(pipe_serial);
7915 %}
7916
7917 instruct unnecessary_membar_acquire() %{
7918 predicate(unnecessary_acquire(n));
7919 match(MemBarAcquire);
7920 ins_cost(0);
7921
7922 format %{ "membar_acquire (elided)" %}
7923
7924 ins_encode %{
7925 __ block_comment("membar_acquire (elided)");
7926 %}
7927
7928 ins_pipe(pipe_class_empty);
7929 %}
7930
7931 instruct membar_acquire() %{
7932 match(MemBarAcquire);
7933 ins_cost(VOLATILE_REF_COST);
7934
7935 format %{ "membar_acquire\n\t"
7936 "dmb ishld" %}
7937
7938 ins_encode %{
7939 __ block_comment("membar_acquire");
7940 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7941 %}
7942
7943 ins_pipe(pipe_serial);
7944 %}
7945
7946
7947 instruct membar_acquire_lock() %{
7948 match(MemBarAcquireLock);
7949 ins_cost(VOLATILE_REF_COST);
7950
7951 format %{ "membar_acquire_lock (elided)" %}
7952
7953 ins_encode %{
7954 __ block_comment("membar_acquire_lock (elided)");
7955 %}
7956
7957 ins_pipe(pipe_serial);
7958 %}
7959
7960 instruct store_fence() %{
7961 match(StoreFence);
7962 ins_cost(VOLATILE_REF_COST);
7963
7964 format %{ "store_fence" %}
7965
7966 ins_encode %{
7967 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7968 %}
7969 ins_pipe(pipe_serial);
7970 %}
7971
7972 instruct unnecessary_membar_release() %{
7973 predicate(unnecessary_release(n));
7974 match(MemBarRelease);
7975 ins_cost(0);
7976
7977 format %{ "membar_release (elided)" %}
7978
7979 ins_encode %{
7980 __ block_comment("membar_release (elided)");
7981 %}
7982 ins_pipe(pipe_serial);
7983 %}
7984
7985 instruct membar_release() %{
7986 match(MemBarRelease);
7987 ins_cost(VOLATILE_REF_COST);
7988
7989 format %{ "membar_release\n\t"
7990 "dmb ishst\n\tdmb ishld" %}
7991
7992 ins_encode %{
7993 __ block_comment("membar_release");
7994 // These will be merged if AlwaysMergeDMB is enabled.
7995 __ membar(Assembler::StoreStore);
7996 __ membar(Assembler::LoadStore);
7997 %}
7998 ins_pipe(pipe_serial);
7999 %}
8000
8001 instruct membar_storestore() %{
8002 match(MemBarStoreStore);
8003 match(StoreStoreFence);
8004 ins_cost(VOLATILE_REF_COST);
8005
8006 format %{ "MEMBAR-store-store" %}
8007
8008 ins_encode %{
8009 __ membar(Assembler::StoreStore);
8010 %}
8011 ins_pipe(pipe_serial);
8012 %}
8013
8014 instruct membar_release_lock() %{
8015 match(MemBarReleaseLock);
8016 ins_cost(VOLATILE_REF_COST);
8017
8018 format %{ "membar_release_lock (elided)" %}
8019
8020 ins_encode %{
8021 __ block_comment("membar_release_lock (elided)");
8022 %}
8023
8024 ins_pipe(pipe_serial);
8025 %}
8026
8027 instruct membar_storeload() %{
8028 match(MemBarStoreLoad);
8029 ins_cost(VOLATILE_REF_COST*100);
8030
8031 format %{ "MEMBAR-store-load\n\t"
8032 "dmb ish" %}
8033
8034 ins_encode %{
8035 __ block_comment("membar_storeload");
8036 __ membar(Assembler::StoreLoad);
8037 %}
8038
8039 ins_pipe(pipe_serial);
8040 %}
8041
8042 instruct unnecessary_membar_volatile() %{
8043 predicate(unnecessary_volatile(n));
8044 match(MemBarVolatile);
8045 ins_cost(0);
8046
8047 format %{ "membar_volatile (elided)" %}
8048
8049 ins_encode %{
8050 __ block_comment("membar_volatile (elided)");
8051 %}
8052
8053 ins_pipe(pipe_serial);
8054 %}
8055
8056 instruct membar_volatile() %{
8057 match(MemBarVolatile);
8058 ins_cost(VOLATILE_REF_COST*100);
8059
8060 format %{ "membar_volatile\n\t"
8061 "dmb ish"%}
8062
8063 ins_encode %{
8064 __ block_comment("membar_volatile");
8065 __ membar(Assembler::StoreLoad);
8066 %}
8067
8068 ins_pipe(pipe_serial);
8069 %}
8070
8071 instruct membar_full() %{
8072 match(MemBarFull);
8073 ins_cost(VOLATILE_REF_COST*100);
8074
8075 format %{ "membar_full\n\t"
8076 "dmb ish" %}
8077 ins_encode %{
8078 __ block_comment("membar_full");
8079 __ membar(Assembler::AnyAny);
8080 %}
8081
8082 ins_pipe(pipe_serial);
8083 %}
8084
8085 // ============================================================================
8086 // Cast/Convert Instructions
8087
8088 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8089 match(Set dst (CastX2P src));
8090
8091 ins_cost(INSN_COST);
8092 format %{ "mov $dst, $src\t# long -> ptr" %}
8093
8094 ins_encode %{
8095 if ($dst$$reg != $src$$reg) {
8096 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8097 }
8098 %}
8099
8100 ins_pipe(ialu_reg);
8101 %}
8102
8103 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8104 match(Set dst (CastP2X src));
8105
8106 ins_cost(INSN_COST);
8107 format %{ "mov $dst, $src\t# ptr -> long" %}
8108
8109 ins_encode %{
8110 if ($dst$$reg != $src$$reg) {
8111 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8112 }
8113 %}
8114
8115 ins_pipe(ialu_reg);
8116 %}
8117
8118 // Convert oop into int for vectors alignment masking
8119 instruct convP2I(iRegINoSp dst, iRegP src) %{
8120 match(Set dst (ConvL2I (CastP2X src)));
8121
8122 ins_cost(INSN_COST);
8123 format %{ "movw $dst, $src\t# ptr -> int" %}
8124 ins_encode %{
8125 __ movw($dst$$Register, $src$$Register);
8126 %}
8127
8128 ins_pipe(ialu_reg);
8129 %}
8130
8131 // Convert compressed oop into int for vectors alignment masking
8132 // in case of 32bit oops (heap < 4Gb).
8133 instruct convN2I(iRegINoSp dst, iRegN src)
8134 %{
8135 predicate(CompressedOops::shift() == 0);
8136 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8137
8138 ins_cost(INSN_COST);
8139 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8140 ins_encode %{
8141 __ movw($dst$$Register, $src$$Register);
8142 %}
8143
8144 ins_pipe(ialu_reg);
8145 %}
8146
8147
8148 // Convert oop pointer into compressed form
8149 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8150 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8151 match(Set dst (EncodeP src));
8152 effect(KILL cr);
8153 ins_cost(INSN_COST * 3);
8154 format %{ "encode_heap_oop $dst, $src" %}
8155 ins_encode %{
8156 Register s = $src$$Register;
8157 Register d = $dst$$Register;
8158 __ encode_heap_oop(d, s);
8159 %}
8160 ins_pipe(ialu_reg);
8161 %}
8162
8163 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8164 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8165 match(Set dst (EncodeP src));
8166 ins_cost(INSN_COST * 3);
8167 format %{ "encode_heap_oop_not_null $dst, $src" %}
8168 ins_encode %{
8169 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8170 %}
8171 ins_pipe(ialu_reg);
8172 %}
8173
8174 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8175 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8176 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8177 match(Set dst (DecodeN src));
8178 ins_cost(INSN_COST * 3);
8179 format %{ "decode_heap_oop $dst, $src" %}
8180 ins_encode %{
8181 Register s = $src$$Register;
8182 Register d = $dst$$Register;
8183 __ decode_heap_oop(d, s);
8184 %}
8185 ins_pipe(ialu_reg);
8186 %}
8187
8188 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8189 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8190 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8191 match(Set dst (DecodeN src));
8192 ins_cost(INSN_COST * 3);
8193 format %{ "decode_heap_oop_not_null $dst, $src" %}
8194 ins_encode %{
8195 Register s = $src$$Register;
8196 Register d = $dst$$Register;
8197 __ decode_heap_oop_not_null(d, s);
8198 %}
8199 ins_pipe(ialu_reg);
8200 %}
8201
8202 // n.b. AArch64 implementations of encode_klass_not_null and
8203 // decode_klass_not_null do not modify the flags register so, unlike
8204 // Intel, we don't kill CR as a side effect here
8205
8206 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8207 match(Set dst (EncodePKlass src));
8208
8209 ins_cost(INSN_COST * 3);
8210 format %{ "encode_klass_not_null $dst,$src" %}
8211
8212 ins_encode %{
8213 Register src_reg = as_Register($src$$reg);
8214 Register dst_reg = as_Register($dst$$reg);
8215 __ encode_klass_not_null(dst_reg, src_reg);
8216 %}
8217
8218 ins_pipe(ialu_reg);
8219 %}
8220
8221 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8222 match(Set dst (DecodeNKlass src));
8223
8224 ins_cost(INSN_COST * 3);
8225 format %{ "decode_klass_not_null $dst,$src" %}
8226
8227 ins_encode %{
8228 Register src_reg = as_Register($src$$reg);
8229 Register dst_reg = as_Register($dst$$reg);
8230 if (dst_reg != src_reg) {
8231 __ decode_klass_not_null(dst_reg, src_reg);
8232 } else {
8233 __ decode_klass_not_null(dst_reg);
8234 }
8235 %}
8236
8237 ins_pipe(ialu_reg);
8238 %}
8239
8240 instruct checkCastPP(iRegPNoSp dst)
8241 %{
8242 match(Set dst (CheckCastPP dst));
8243
8244 size(0);
8245 format %{ "# checkcastPP of $dst" %}
8246 ins_encode(/* empty encoding */);
8247 ins_pipe(pipe_class_empty);
8248 %}
8249
8250 instruct castPP(iRegPNoSp dst)
8251 %{
8252 match(Set dst (CastPP dst));
8253
8254 size(0);
8255 format %{ "# castPP of $dst" %}
8256 ins_encode(/* empty encoding */);
8257 ins_pipe(pipe_class_empty);
8258 %}
8259
8260 instruct castII(iRegI dst)
8261 %{
8262 predicate(VerifyConstraintCasts == 0);
8263 match(Set dst (CastII dst));
8264
8265 size(0);
8266 format %{ "# castII of $dst" %}
8267 ins_encode(/* empty encoding */);
8268 ins_cost(0);
8269 ins_pipe(pipe_class_empty);
8270 %}
8271
8272 instruct castII_checked(iRegI dst, rFlagsReg cr)
8273 %{
8274 predicate(VerifyConstraintCasts > 0);
8275 match(Set dst (CastII dst));
8276 effect(KILL cr);
8277
8278 format %{ "# castII_checked of $dst" %}
8279 ins_encode %{
8280 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8281 %}
8282 ins_pipe(pipe_slow);
8283 %}
8284
8285 instruct castLL(iRegL dst)
8286 %{
8287 predicate(VerifyConstraintCasts == 0);
8288 match(Set dst (CastLL dst));
8289
8290 size(0);
8291 format %{ "# castLL of $dst" %}
8292 ins_encode(/* empty encoding */);
8293 ins_cost(0);
8294 ins_pipe(pipe_class_empty);
8295 %}
8296
8297 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8298 %{
8299 predicate(VerifyConstraintCasts > 0);
8300 match(Set dst (CastLL dst));
8301 effect(KILL cr);
8302
8303 format %{ "# castLL_checked of $dst" %}
8304 ins_encode %{
8305 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8306 %}
8307 ins_pipe(pipe_slow);
8308 %}
8309
8310 instruct castHH(vRegF dst)
8311 %{
8312 match(Set dst (CastHH dst));
8313 size(0);
8314 format %{ "# castHH of $dst" %}
8315 ins_encode(/* empty encoding */);
8316 ins_cost(0);
8317 ins_pipe(pipe_class_empty);
8318 %}
8319
8320 instruct castFF(vRegF dst)
8321 %{
8322 match(Set dst (CastFF dst));
8323
8324 size(0);
8325 format %{ "# castFF of $dst" %}
8326 ins_encode(/* empty encoding */);
8327 ins_cost(0);
8328 ins_pipe(pipe_class_empty);
8329 %}
8330
8331 instruct castDD(vRegD dst)
8332 %{
8333 match(Set dst (CastDD dst));
8334
8335 size(0);
8336 format %{ "# castDD of $dst" %}
8337 ins_encode(/* empty encoding */);
8338 ins_cost(0);
8339 ins_pipe(pipe_class_empty);
8340 %}
8341
8342 instruct castVV(vReg dst)
8343 %{
8344 match(Set dst (CastVV dst));
8345
8346 size(0);
8347 format %{ "# castVV of $dst" %}
8348 ins_encode(/* empty encoding */);
8349 ins_cost(0);
8350 ins_pipe(pipe_class_empty);
8351 %}
8352
8353 instruct castVVMask(pRegGov dst)
8354 %{
8355 match(Set dst (CastVV dst));
8356
8357 size(0);
8358 format %{ "# castVV of $dst" %}
8359 ins_encode(/* empty encoding */);
8360 ins_cost(0);
8361 ins_pipe(pipe_class_empty);
8362 %}
8363
8364 // Manifest a CmpU result in an integer register.
8365 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8366 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8367 %{
8368 match(Set dst (CmpU3 src1 src2));
8369 effect(KILL flags);
8370
8371 ins_cost(INSN_COST * 3);
8372 format %{
8373 "cmpw $src1, $src2\n\t"
8374 "csetw $dst, ne\n\t"
8375 "cnegw $dst, lo\t# CmpU3(reg)"
8376 %}
8377 ins_encode %{
8378 __ cmpw($src1$$Register, $src2$$Register);
8379 __ csetw($dst$$Register, Assembler::NE);
8380 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8381 %}
8382
8383 ins_pipe(pipe_class_default);
8384 %}
8385
8386 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8387 %{
8388 match(Set dst (CmpU3 src1 src2));
8389 effect(KILL flags);
8390
8391 ins_cost(INSN_COST * 3);
8392 format %{
8393 "subsw zr, $src1, $src2\n\t"
8394 "csetw $dst, ne\n\t"
8395 "cnegw $dst, lo\t# CmpU3(imm)"
8396 %}
8397 ins_encode %{
8398 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8399 __ csetw($dst$$Register, Assembler::NE);
8400 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8401 %}
8402
8403 ins_pipe(pipe_class_default);
8404 %}
8405
8406 // Manifest a CmpUL result in an integer register.
8407 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8408 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8409 %{
8410 match(Set dst (CmpUL3 src1 src2));
8411 effect(KILL flags);
8412
8413 ins_cost(INSN_COST * 3);
8414 format %{
8415 "cmp $src1, $src2\n\t"
8416 "csetw $dst, ne\n\t"
8417 "cnegw $dst, lo\t# CmpUL3(reg)"
8418 %}
8419 ins_encode %{
8420 __ cmp($src1$$Register, $src2$$Register);
8421 __ csetw($dst$$Register, Assembler::NE);
8422 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8423 %}
8424
8425 ins_pipe(pipe_class_default);
8426 %}
8427
8428 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8429 %{
8430 match(Set dst (CmpUL3 src1 src2));
8431 effect(KILL flags);
8432
8433 ins_cost(INSN_COST * 3);
8434 format %{
8435 "subs zr, $src1, $src2\n\t"
8436 "csetw $dst, ne\n\t"
8437 "cnegw $dst, lo\t# CmpUL3(imm)"
8438 %}
8439 ins_encode %{
8440 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8441 __ csetw($dst$$Register, Assembler::NE);
8442 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8443 %}
8444
8445 ins_pipe(pipe_class_default);
8446 %}
8447
8448 // Manifest a CmpL result in an integer register.
8449 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8450 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8451 %{
8452 match(Set dst (CmpL3 src1 src2));
8453 effect(KILL flags);
8454
8455 ins_cost(INSN_COST * 3);
8456 format %{
8457 "cmp $src1, $src2\n\t"
8458 "csetw $dst, ne\n\t"
8459 "cnegw $dst, lt\t# CmpL3(reg)"
8460 %}
8461 ins_encode %{
8462 __ cmp($src1$$Register, $src2$$Register);
8463 __ csetw($dst$$Register, Assembler::NE);
8464 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8465 %}
8466
8467 ins_pipe(pipe_class_default);
8468 %}
8469
8470 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8471 %{
8472 match(Set dst (CmpL3 src1 src2));
8473 effect(KILL flags);
8474
8475 ins_cost(INSN_COST * 3);
8476 format %{
8477 "subs zr, $src1, $src2\n\t"
8478 "csetw $dst, ne\n\t"
8479 "cnegw $dst, lt\t# CmpL3(imm)"
8480 %}
8481 ins_encode %{
8482 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8483 __ csetw($dst$$Register, Assembler::NE);
8484 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8485 %}
8486
8487 ins_pipe(pipe_class_default);
8488 %}
8489
8490 // ============================================================================
8491 // Conditional Move Instructions
8492
8493 // n.b. we have identical rules for both a signed compare op (cmpOp)
8494 // and an unsigned compare op (cmpOpU). it would be nice if we could
8495 // define an op class which merged both inputs and use it to type the
8496 // argument to a single rule. unfortunatelyt his fails because the
8497 // opclass does not live up to the COND_INTER interface of its
8498 // component operands. When the generic code tries to negate the
8499 // operand it ends up running the generci Machoper::negate method
8500 // which throws a ShouldNotHappen. So, we have to provide two flavours
8501 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8502
8503 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8504 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8505
8506 ins_cost(INSN_COST * 2);
8507 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8508
8509 ins_encode %{
8510 __ cselw(as_Register($dst$$reg),
8511 as_Register($src2$$reg),
8512 as_Register($src1$$reg),
8513 (Assembler::Condition)$cmp$$cmpcode);
8514 %}
8515
8516 ins_pipe(icond_reg_reg);
8517 %}
8518
8519 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8520 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8521
8522 ins_cost(INSN_COST * 2);
8523 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8524
8525 ins_encode %{
8526 __ cselw(as_Register($dst$$reg),
8527 as_Register($src2$$reg),
8528 as_Register($src1$$reg),
8529 (Assembler::Condition)$cmp$$cmpcode);
8530 %}
8531
8532 ins_pipe(icond_reg_reg);
8533 %}
8534
8535 // special cases where one arg is zero
8536
8537 // n.b. this is selected in preference to the rule above because it
8538 // avoids loading constant 0 into a source register
8539
8540 // TODO
8541 // we ought only to be able to cull one of these variants as the ideal
8542 // transforms ought always to order the zero consistently (to left/right?)
8543
8544 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8545 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8546
8547 ins_cost(INSN_COST * 2);
8548 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8549
8550 ins_encode %{
8551 __ cselw(as_Register($dst$$reg),
8552 as_Register($src$$reg),
8553 zr,
8554 (Assembler::Condition)$cmp$$cmpcode);
8555 %}
8556
8557 ins_pipe(icond_reg);
8558 %}
8559
8560 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8561 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8562
8563 ins_cost(INSN_COST * 2);
8564 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8565
8566 ins_encode %{
8567 __ cselw(as_Register($dst$$reg),
8568 as_Register($src$$reg),
8569 zr,
8570 (Assembler::Condition)$cmp$$cmpcode);
8571 %}
8572
8573 ins_pipe(icond_reg);
8574 %}
8575
8576 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8577 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8578
8579 ins_cost(INSN_COST * 2);
8580 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8581
8582 ins_encode %{
8583 __ cselw(as_Register($dst$$reg),
8584 zr,
8585 as_Register($src$$reg),
8586 (Assembler::Condition)$cmp$$cmpcode);
8587 %}
8588
8589 ins_pipe(icond_reg);
8590 %}
8591
8592 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8593 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8594
8595 ins_cost(INSN_COST * 2);
8596 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8597
8598 ins_encode %{
8599 __ cselw(as_Register($dst$$reg),
8600 zr,
8601 as_Register($src$$reg),
8602 (Assembler::Condition)$cmp$$cmpcode);
8603 %}
8604
8605 ins_pipe(icond_reg);
8606 %}
8607
8608 // special case for creating a boolean 0 or 1
8609
8610 // n.b. this is selected in preference to the rule above because it
8611 // avoids loading constants 0 and 1 into a source register
8612
8613 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8614 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8615
8616 ins_cost(INSN_COST * 2);
8617 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8618
8619 ins_encode %{
8620 // equivalently
8621 // cset(as_Register($dst$$reg),
8622 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8623 __ csincw(as_Register($dst$$reg),
8624 zr,
8625 zr,
8626 (Assembler::Condition)$cmp$$cmpcode);
8627 %}
8628
8629 ins_pipe(icond_none);
8630 %}
8631
8632 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8633 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8634
8635 ins_cost(INSN_COST * 2);
8636 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8637
8638 ins_encode %{
8639 // equivalently
8640 // cset(as_Register($dst$$reg),
8641 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8642 __ csincw(as_Register($dst$$reg),
8643 zr,
8644 zr,
8645 (Assembler::Condition)$cmp$$cmpcode);
8646 %}
8647
8648 ins_pipe(icond_none);
8649 %}
8650
8651 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8652 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8653
8654 ins_cost(INSN_COST * 2);
8655 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8656
8657 ins_encode %{
8658 __ csel(as_Register($dst$$reg),
8659 as_Register($src2$$reg),
8660 as_Register($src1$$reg),
8661 (Assembler::Condition)$cmp$$cmpcode);
8662 %}
8663
8664 ins_pipe(icond_reg_reg);
8665 %}
8666
8667 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8668 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8669
8670 ins_cost(INSN_COST * 2);
8671 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8672
8673 ins_encode %{
8674 __ csel(as_Register($dst$$reg),
8675 as_Register($src2$$reg),
8676 as_Register($src1$$reg),
8677 (Assembler::Condition)$cmp$$cmpcode);
8678 %}
8679
8680 ins_pipe(icond_reg_reg);
8681 %}
8682
8683 // special cases where one arg is zero
8684
8685 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8686 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8687
8688 ins_cost(INSN_COST * 2);
8689 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8690
8691 ins_encode %{
8692 __ csel(as_Register($dst$$reg),
8693 zr,
8694 as_Register($src$$reg),
8695 (Assembler::Condition)$cmp$$cmpcode);
8696 %}
8697
8698 ins_pipe(icond_reg);
8699 %}
8700
8701 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8702 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8703
8704 ins_cost(INSN_COST * 2);
8705 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8706
8707 ins_encode %{
8708 __ csel(as_Register($dst$$reg),
8709 zr,
8710 as_Register($src$$reg),
8711 (Assembler::Condition)$cmp$$cmpcode);
8712 %}
8713
8714 ins_pipe(icond_reg);
8715 %}
8716
8717 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8718 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8719
8720 ins_cost(INSN_COST * 2);
8721 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8722
8723 ins_encode %{
8724 __ csel(as_Register($dst$$reg),
8725 as_Register($src$$reg),
8726 zr,
8727 (Assembler::Condition)$cmp$$cmpcode);
8728 %}
8729
8730 ins_pipe(icond_reg);
8731 %}
8732
8733 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8734 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8735
8736 ins_cost(INSN_COST * 2);
8737 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8738
8739 ins_encode %{
8740 __ csel(as_Register($dst$$reg),
8741 as_Register($src$$reg),
8742 zr,
8743 (Assembler::Condition)$cmp$$cmpcode);
8744 %}
8745
8746 ins_pipe(icond_reg);
8747 %}
8748
8749 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8750 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8751
8752 ins_cost(INSN_COST * 2);
8753 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8754
8755 ins_encode %{
8756 __ csel(as_Register($dst$$reg),
8757 as_Register($src2$$reg),
8758 as_Register($src1$$reg),
8759 (Assembler::Condition)$cmp$$cmpcode);
8760 %}
8761
8762 ins_pipe(icond_reg_reg);
8763 %}
8764
8765 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8766 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8767
8768 ins_cost(INSN_COST * 2);
8769 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8770
8771 ins_encode %{
8772 __ csel(as_Register($dst$$reg),
8773 as_Register($src2$$reg),
8774 as_Register($src1$$reg),
8775 (Assembler::Condition)$cmp$$cmpcode);
8776 %}
8777
8778 ins_pipe(icond_reg_reg);
8779 %}
8780
8781 // special cases where one arg is zero
8782
8783 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8784 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8785
8786 ins_cost(INSN_COST * 2);
8787 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8788
8789 ins_encode %{
8790 __ csel(as_Register($dst$$reg),
8791 zr,
8792 as_Register($src$$reg),
8793 (Assembler::Condition)$cmp$$cmpcode);
8794 %}
8795
8796 ins_pipe(icond_reg);
8797 %}
8798
8799 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8800 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8801
8802 ins_cost(INSN_COST * 2);
8803 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8804
8805 ins_encode %{
8806 __ csel(as_Register($dst$$reg),
8807 zr,
8808 as_Register($src$$reg),
8809 (Assembler::Condition)$cmp$$cmpcode);
8810 %}
8811
8812 ins_pipe(icond_reg);
8813 %}
8814
8815 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8816 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8817
8818 ins_cost(INSN_COST * 2);
8819 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8820
8821 ins_encode %{
8822 __ csel(as_Register($dst$$reg),
8823 as_Register($src$$reg),
8824 zr,
8825 (Assembler::Condition)$cmp$$cmpcode);
8826 %}
8827
8828 ins_pipe(icond_reg);
8829 %}
8830
8831 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8832 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8833
8834 ins_cost(INSN_COST * 2);
8835 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8836
8837 ins_encode %{
8838 __ csel(as_Register($dst$$reg),
8839 as_Register($src$$reg),
8840 zr,
8841 (Assembler::Condition)$cmp$$cmpcode);
8842 %}
8843
8844 ins_pipe(icond_reg);
8845 %}
8846
8847 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8848 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8849
8850 ins_cost(INSN_COST * 2);
8851 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8852
8853 ins_encode %{
8854 __ cselw(as_Register($dst$$reg),
8855 as_Register($src2$$reg),
8856 as_Register($src1$$reg),
8857 (Assembler::Condition)$cmp$$cmpcode);
8858 %}
8859
8860 ins_pipe(icond_reg_reg);
8861 %}
8862
8863 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8864 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8865
8866 ins_cost(INSN_COST * 2);
8867 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8868
8869 ins_encode %{
8870 __ cselw(as_Register($dst$$reg),
8871 as_Register($src2$$reg),
8872 as_Register($src1$$reg),
8873 (Assembler::Condition)$cmp$$cmpcode);
8874 %}
8875
8876 ins_pipe(icond_reg_reg);
8877 %}
8878
8879 // special cases where one arg is zero
8880
8881 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8882 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8883
8884 ins_cost(INSN_COST * 2);
8885 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8886
8887 ins_encode %{
8888 __ cselw(as_Register($dst$$reg),
8889 zr,
8890 as_Register($src$$reg),
8891 (Assembler::Condition)$cmp$$cmpcode);
8892 %}
8893
8894 ins_pipe(icond_reg);
8895 %}
8896
8897 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8898 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8899
8900 ins_cost(INSN_COST * 2);
8901 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8902
8903 ins_encode %{
8904 __ cselw(as_Register($dst$$reg),
8905 zr,
8906 as_Register($src$$reg),
8907 (Assembler::Condition)$cmp$$cmpcode);
8908 %}
8909
8910 ins_pipe(icond_reg);
8911 %}
8912
8913 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8914 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8915
8916 ins_cost(INSN_COST * 2);
8917 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8918
8919 ins_encode %{
8920 __ cselw(as_Register($dst$$reg),
8921 as_Register($src$$reg),
8922 zr,
8923 (Assembler::Condition)$cmp$$cmpcode);
8924 %}
8925
8926 ins_pipe(icond_reg);
8927 %}
8928
8929 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8930 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8931
8932 ins_cost(INSN_COST * 2);
8933 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8934
8935 ins_encode %{
8936 __ cselw(as_Register($dst$$reg),
8937 as_Register($src$$reg),
8938 zr,
8939 (Assembler::Condition)$cmp$$cmpcode);
8940 %}
8941
8942 ins_pipe(icond_reg);
8943 %}
8944
8945 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8946 %{
8947 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8948
8949 ins_cost(INSN_COST * 3);
8950
8951 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8952 ins_encode %{
8953 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8954 __ fcsels(as_FloatRegister($dst$$reg),
8955 as_FloatRegister($src2$$reg),
8956 as_FloatRegister($src1$$reg),
8957 cond);
8958 %}
8959
8960 ins_pipe(fp_cond_reg_reg_s);
8961 %}
8962
8963 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8964 %{
8965 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8966
8967 ins_cost(INSN_COST * 3);
8968
8969 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8970 ins_encode %{
8971 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8972 __ fcsels(as_FloatRegister($dst$$reg),
8973 as_FloatRegister($src2$$reg),
8974 as_FloatRegister($src1$$reg),
8975 cond);
8976 %}
8977
8978 ins_pipe(fp_cond_reg_reg_s);
8979 %}
8980
8981 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8982 %{
8983 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8984
8985 ins_cost(INSN_COST * 3);
8986
8987 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8988 ins_encode %{
8989 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8990 __ fcseld(as_FloatRegister($dst$$reg),
8991 as_FloatRegister($src2$$reg),
8992 as_FloatRegister($src1$$reg),
8993 cond);
8994 %}
8995
8996 ins_pipe(fp_cond_reg_reg_d);
8997 %}
8998
8999 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9000 %{
9001 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9002
9003 ins_cost(INSN_COST * 3);
9004
9005 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9006 ins_encode %{
9007 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9008 __ fcseld(as_FloatRegister($dst$$reg),
9009 as_FloatRegister($src2$$reg),
9010 as_FloatRegister($src1$$reg),
9011 cond);
9012 %}
9013
9014 ins_pipe(fp_cond_reg_reg_d);
9015 %}
9016
9017 // ============================================================================
9018 // Arithmetic Instructions
9019 //
9020
9021 // Integer Addition
9022
9023 // TODO
9024 // these currently employ operations which do not set CR and hence are
9025 // not flagged as killing CR but we would like to isolate the cases
9026 // where we want to set flags from those where we don't. need to work
9027 // out how to do that.
9028
9029 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9030 match(Set dst (AddI src1 src2));
9031
9032 ins_cost(INSN_COST);
9033 format %{ "addw $dst, $src1, $src2" %}
9034
9035 ins_encode %{
9036 __ addw(as_Register($dst$$reg),
9037 as_Register($src1$$reg),
9038 as_Register($src2$$reg));
9039 %}
9040
9041 ins_pipe(ialu_reg_reg);
9042 %}
9043
9044 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9045 match(Set dst (AddI src1 src2));
9046
9047 ins_cost(INSN_COST);
9048 format %{ "addw $dst, $src1, $src2" %}
9049
9050 // use opcode to indicate that this is an add not a sub
9051 opcode(0x0);
9052
9053 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9054
9055 ins_pipe(ialu_reg_imm);
9056 %}
9057
9058 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9059 match(Set dst (AddI (ConvL2I src1) src2));
9060
9061 ins_cost(INSN_COST);
9062 format %{ "addw $dst, $src1, $src2" %}
9063
9064 // use opcode to indicate that this is an add not a sub
9065 opcode(0x0);
9066
9067 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9068
9069 ins_pipe(ialu_reg_imm);
9070 %}
9071
9072 // Pointer Addition
9073 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9074 match(Set dst (AddP src1 src2));
9075
9076 ins_cost(INSN_COST);
9077 format %{ "add $dst, $src1, $src2\t# ptr" %}
9078
9079 ins_encode %{
9080 __ add(as_Register($dst$$reg),
9081 as_Register($src1$$reg),
9082 as_Register($src2$$reg));
9083 %}
9084
9085 ins_pipe(ialu_reg_reg);
9086 %}
9087
9088 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9089 match(Set dst (AddP src1 (ConvI2L src2)));
9090
9091 ins_cost(1.9 * INSN_COST);
9092 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9093
9094 ins_encode %{
9095 __ add(as_Register($dst$$reg),
9096 as_Register($src1$$reg),
9097 as_Register($src2$$reg), ext::sxtw);
9098 %}
9099
9100 ins_pipe(ialu_reg_reg);
9101 %}
9102
9103 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9104 match(Set dst (AddP src1 (LShiftL src2 scale)));
9105
9106 ins_cost(1.9 * INSN_COST);
9107 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9108
9109 ins_encode %{
9110 __ lea(as_Register($dst$$reg),
9111 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9112 Address::lsl($scale$$constant)));
9113 %}
9114
9115 ins_pipe(ialu_reg_reg_shift);
9116 %}
9117
9118 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9119 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9120
9121 ins_cost(1.9 * INSN_COST);
9122 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9123
9124 ins_encode %{
9125 __ lea(as_Register($dst$$reg),
9126 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9127 Address::sxtw($scale$$constant)));
9128 %}
9129
9130 ins_pipe(ialu_reg_reg_shift);
9131 %}
9132
9133 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9134 match(Set dst (LShiftL (ConvI2L src) scale));
9135
9136 ins_cost(INSN_COST);
9137 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9138
9139 ins_encode %{
9140 __ sbfiz(as_Register($dst$$reg),
9141 as_Register($src$$reg),
9142 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9143 %}
9144
9145 ins_pipe(ialu_reg_shift);
9146 %}
9147
9148 // Pointer Immediate Addition
9149 // n.b. this needs to be more expensive than using an indirect memory
9150 // operand
9151 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9152 match(Set dst (AddP src1 src2));
9153
9154 ins_cost(INSN_COST);
9155 format %{ "add $dst, $src1, $src2\t# ptr" %}
9156
9157 // use opcode to indicate that this is an add not a sub
9158 opcode(0x0);
9159
9160 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9161
9162 ins_pipe(ialu_reg_imm);
9163 %}
9164
9165 // Long Addition
9166 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9167
9168 match(Set dst (AddL src1 src2));
9169
9170 ins_cost(INSN_COST);
9171 format %{ "add $dst, $src1, $src2" %}
9172
9173 ins_encode %{
9174 __ add(as_Register($dst$$reg),
9175 as_Register($src1$$reg),
9176 as_Register($src2$$reg));
9177 %}
9178
9179 ins_pipe(ialu_reg_reg);
9180 %}
9181
9182 // No constant pool entries requiredLong Immediate Addition.
9183 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9184 match(Set dst (AddL src1 src2));
9185
9186 ins_cost(INSN_COST);
9187 format %{ "add $dst, $src1, $src2" %}
9188
9189 // use opcode to indicate that this is an add not a sub
9190 opcode(0x0);
9191
9192 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9193
9194 ins_pipe(ialu_reg_imm);
9195 %}
9196
9197 // Integer Subtraction
9198 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9199 match(Set dst (SubI src1 src2));
9200
9201 ins_cost(INSN_COST);
9202 format %{ "subw $dst, $src1, $src2" %}
9203
9204 ins_encode %{
9205 __ subw(as_Register($dst$$reg),
9206 as_Register($src1$$reg),
9207 as_Register($src2$$reg));
9208 %}
9209
9210 ins_pipe(ialu_reg_reg);
9211 %}
9212
9213 // Immediate Subtraction
9214 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9215 match(Set dst (SubI src1 src2));
9216
9217 ins_cost(INSN_COST);
9218 format %{ "subw $dst, $src1, $src2" %}
9219
9220 // use opcode to indicate that this is a sub not an add
9221 opcode(0x1);
9222
9223 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9224
9225 ins_pipe(ialu_reg_imm);
9226 %}
9227
9228 // Long Subtraction
9229 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9230
9231 match(Set dst (SubL src1 src2));
9232
9233 ins_cost(INSN_COST);
9234 format %{ "sub $dst, $src1, $src2" %}
9235
9236 ins_encode %{
9237 __ sub(as_Register($dst$$reg),
9238 as_Register($src1$$reg),
9239 as_Register($src2$$reg));
9240 %}
9241
9242 ins_pipe(ialu_reg_reg);
9243 %}
9244
9245 // No constant pool entries requiredLong Immediate Subtraction.
9246 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9247 match(Set dst (SubL src1 src2));
9248
9249 ins_cost(INSN_COST);
9250 format %{ "sub$dst, $src1, $src2" %}
9251
9252 // use opcode to indicate that this is a sub not an add
9253 opcode(0x1);
9254
9255 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9256
9257 ins_pipe(ialu_reg_imm);
9258 %}
9259
9260 // Integer Negation (special case for sub)
9261
9262 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9263 match(Set dst (SubI zero src));
9264
9265 ins_cost(INSN_COST);
9266 format %{ "negw $dst, $src\t# int" %}
9267
9268 ins_encode %{
9269 __ negw(as_Register($dst$$reg),
9270 as_Register($src$$reg));
9271 %}
9272
9273 ins_pipe(ialu_reg);
9274 %}
9275
9276 // Long Negation
9277
9278 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9279 match(Set dst (SubL zero src));
9280
9281 ins_cost(INSN_COST);
9282 format %{ "neg $dst, $src\t# long" %}
9283
9284 ins_encode %{
9285 __ neg(as_Register($dst$$reg),
9286 as_Register($src$$reg));
9287 %}
9288
9289 ins_pipe(ialu_reg);
9290 %}
9291
9292 // Integer Multiply
9293
9294 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9295 match(Set dst (MulI src1 src2));
9296
9297 ins_cost(INSN_COST * 3);
9298 format %{ "mulw $dst, $src1, $src2" %}
9299
9300 ins_encode %{
9301 __ mulw(as_Register($dst$$reg),
9302 as_Register($src1$$reg),
9303 as_Register($src2$$reg));
9304 %}
9305
9306 ins_pipe(imul_reg_reg);
9307 %}
9308
9309 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9310 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9311
9312 ins_cost(INSN_COST * 3);
9313 format %{ "smull $dst, $src1, $src2" %}
9314
9315 ins_encode %{
9316 __ smull(as_Register($dst$$reg),
9317 as_Register($src1$$reg),
9318 as_Register($src2$$reg));
9319 %}
9320
9321 ins_pipe(imul_reg_reg);
9322 %}
9323
9324 // Long Multiply
9325
9326 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9327 match(Set dst (MulL src1 src2));
9328
9329 ins_cost(INSN_COST * 5);
9330 format %{ "mul $dst, $src1, $src2" %}
9331
9332 ins_encode %{
9333 __ mul(as_Register($dst$$reg),
9334 as_Register($src1$$reg),
9335 as_Register($src2$$reg));
9336 %}
9337
9338 ins_pipe(lmul_reg_reg);
9339 %}
9340
9341 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9342 %{
9343 match(Set dst (MulHiL src1 src2));
9344
9345 ins_cost(INSN_COST * 7);
9346 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9347
9348 ins_encode %{
9349 __ smulh(as_Register($dst$$reg),
9350 as_Register($src1$$reg),
9351 as_Register($src2$$reg));
9352 %}
9353
9354 ins_pipe(lmul_reg_reg);
9355 %}
9356
9357 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9358 %{
9359 match(Set dst (UMulHiL src1 src2));
9360
9361 ins_cost(INSN_COST * 7);
9362 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9363
9364 ins_encode %{
9365 __ umulh(as_Register($dst$$reg),
9366 as_Register($src1$$reg),
9367 as_Register($src2$$reg));
9368 %}
9369
9370 ins_pipe(lmul_reg_reg);
9371 %}
9372
9373 // Combined Integer Multiply & Add/Sub
9374
9375 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9376 match(Set dst (AddI src3 (MulI src1 src2)));
9377
9378 ins_cost(INSN_COST * 3);
9379 format %{ "madd $dst, $src1, $src2, $src3" %}
9380
9381 ins_encode %{
9382 __ maddw(as_Register($dst$$reg),
9383 as_Register($src1$$reg),
9384 as_Register($src2$$reg),
9385 as_Register($src3$$reg));
9386 %}
9387
9388 ins_pipe(imac_reg_reg);
9389 %}
9390
9391 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9392 match(Set dst (SubI src3 (MulI src1 src2)));
9393
9394 ins_cost(INSN_COST * 3);
9395 format %{ "msub $dst, $src1, $src2, $src3" %}
9396
9397 ins_encode %{
9398 __ msubw(as_Register($dst$$reg),
9399 as_Register($src1$$reg),
9400 as_Register($src2$$reg),
9401 as_Register($src3$$reg));
9402 %}
9403
9404 ins_pipe(imac_reg_reg);
9405 %}
9406
9407 // Combined Integer Multiply & Neg
9408
9409 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9410 match(Set dst (MulI (SubI zero src1) src2));
9411
9412 ins_cost(INSN_COST * 3);
9413 format %{ "mneg $dst, $src1, $src2" %}
9414
9415 ins_encode %{
9416 __ mnegw(as_Register($dst$$reg),
9417 as_Register($src1$$reg),
9418 as_Register($src2$$reg));
9419 %}
9420
9421 ins_pipe(imac_reg_reg);
9422 %}
9423
9424 // Combined Long Multiply & Add/Sub
9425
9426 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9427 match(Set dst (AddL src3 (MulL src1 src2)));
9428
9429 ins_cost(INSN_COST * 5);
9430 format %{ "madd $dst, $src1, $src2, $src3" %}
9431
9432 ins_encode %{
9433 __ madd(as_Register($dst$$reg),
9434 as_Register($src1$$reg),
9435 as_Register($src2$$reg),
9436 as_Register($src3$$reg));
9437 %}
9438
9439 ins_pipe(lmac_reg_reg);
9440 %}
9441
9442 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9443 match(Set dst (SubL src3 (MulL src1 src2)));
9444
9445 ins_cost(INSN_COST * 5);
9446 format %{ "msub $dst, $src1, $src2, $src3" %}
9447
9448 ins_encode %{
9449 __ msub(as_Register($dst$$reg),
9450 as_Register($src1$$reg),
9451 as_Register($src2$$reg),
9452 as_Register($src3$$reg));
9453 %}
9454
9455 ins_pipe(lmac_reg_reg);
9456 %}
9457
9458 // Combined Long Multiply & Neg
9459
9460 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9461 match(Set dst (MulL (SubL zero src1) src2));
9462
9463 ins_cost(INSN_COST * 5);
9464 format %{ "mneg $dst, $src1, $src2" %}
9465
9466 ins_encode %{
9467 __ mneg(as_Register($dst$$reg),
9468 as_Register($src1$$reg),
9469 as_Register($src2$$reg));
9470 %}
9471
9472 ins_pipe(lmac_reg_reg);
9473 %}
9474
9475 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9476
9477 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9478 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9479
9480 ins_cost(INSN_COST * 3);
9481 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9482
9483 ins_encode %{
9484 __ smaddl(as_Register($dst$$reg),
9485 as_Register($src1$$reg),
9486 as_Register($src2$$reg),
9487 as_Register($src3$$reg));
9488 %}
9489
9490 ins_pipe(imac_reg_reg);
9491 %}
9492
9493 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9494 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9495
9496 ins_cost(INSN_COST * 3);
9497 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9498
9499 ins_encode %{
9500 __ smsubl(as_Register($dst$$reg),
9501 as_Register($src1$$reg),
9502 as_Register($src2$$reg),
9503 as_Register($src3$$reg));
9504 %}
9505
9506 ins_pipe(imac_reg_reg);
9507 %}
9508
9509 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9510 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9511
9512 ins_cost(INSN_COST * 3);
9513 format %{ "smnegl $dst, $src1, $src2" %}
9514
9515 ins_encode %{
9516 __ smnegl(as_Register($dst$$reg),
9517 as_Register($src1$$reg),
9518 as_Register($src2$$reg));
9519 %}
9520
9521 ins_pipe(imac_reg_reg);
9522 %}
9523
9524 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9525
9526 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9527 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9528
9529 ins_cost(INSN_COST * 5);
9530 format %{ "mulw rscratch1, $src1, $src2\n\t"
9531 "maddw $dst, $src3, $src4, rscratch1" %}
9532
9533 ins_encode %{
9534 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9535 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9536
9537 ins_pipe(imac_reg_reg);
9538 %}
9539
9540 // Integer Divide
9541
9542 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9543 match(Set dst (DivI src1 src2));
9544
9545 ins_cost(INSN_COST * 19);
9546 format %{ "sdivw $dst, $src1, $src2" %}
9547
9548 ins_encode(aarch64_enc_divw(dst, src1, src2));
9549 ins_pipe(idiv_reg_reg);
9550 %}
9551
9552 // Long Divide
9553
9554 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9555 match(Set dst (DivL src1 src2));
9556
9557 ins_cost(INSN_COST * 35);
9558 format %{ "sdiv $dst, $src1, $src2" %}
9559
9560 ins_encode(aarch64_enc_div(dst, src1, src2));
9561 ins_pipe(ldiv_reg_reg);
9562 %}
9563
9564 // Integer Remainder
9565
9566 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9567 match(Set dst (ModI src1 src2));
9568
9569 ins_cost(INSN_COST * 22);
9570 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9571 "msubw $dst, rscratch1, $src2, $src1" %}
9572
9573 ins_encode(aarch64_enc_modw(dst, src1, src2));
9574 ins_pipe(idiv_reg_reg);
9575 %}
9576
9577 // Long Remainder
9578
9579 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9580 match(Set dst (ModL src1 src2));
9581
9582 ins_cost(INSN_COST * 38);
9583 format %{ "sdiv rscratch1, $src1, $src2\n"
9584 "msub $dst, rscratch1, $src2, $src1" %}
9585
9586 ins_encode(aarch64_enc_mod(dst, src1, src2));
9587 ins_pipe(ldiv_reg_reg);
9588 %}
9589
9590 // Unsigned Integer Divide
9591
9592 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9593 match(Set dst (UDivI src1 src2));
9594
9595 ins_cost(INSN_COST * 19);
9596 format %{ "udivw $dst, $src1, $src2" %}
9597
9598 ins_encode %{
9599 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9600 %}
9601
9602 ins_pipe(idiv_reg_reg);
9603 %}
9604
9605 // Unsigned Long Divide
9606
9607 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9608 match(Set dst (UDivL src1 src2));
9609
9610 ins_cost(INSN_COST * 35);
9611 format %{ "udiv $dst, $src1, $src2" %}
9612
9613 ins_encode %{
9614 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9615 %}
9616
9617 ins_pipe(ldiv_reg_reg);
9618 %}
9619
9620 // Unsigned Integer Remainder
9621
9622 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9623 match(Set dst (UModI src1 src2));
9624
9625 ins_cost(INSN_COST * 22);
9626 format %{ "udivw rscratch1, $src1, $src2\n\t"
9627 "msubw $dst, rscratch1, $src2, $src1" %}
9628
9629 ins_encode %{
9630 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9631 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9632 %}
9633
9634 ins_pipe(idiv_reg_reg);
9635 %}
9636
9637 // Unsigned Long Remainder
9638
9639 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9640 match(Set dst (UModL src1 src2));
9641
9642 ins_cost(INSN_COST * 38);
9643 format %{ "udiv rscratch1, $src1, $src2\n"
9644 "msub $dst, rscratch1, $src2, $src1" %}
9645
9646 ins_encode %{
9647 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9648 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9649 %}
9650
9651 ins_pipe(ldiv_reg_reg);
9652 %}
9653
9654 // Integer Shifts
9655
9656 // Shift Left Register
9657 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9658 match(Set dst (LShiftI src1 src2));
9659
9660 ins_cost(INSN_COST * 2);
9661 format %{ "lslvw $dst, $src1, $src2" %}
9662
9663 ins_encode %{
9664 __ lslvw(as_Register($dst$$reg),
9665 as_Register($src1$$reg),
9666 as_Register($src2$$reg));
9667 %}
9668
9669 ins_pipe(ialu_reg_reg_vshift);
9670 %}
9671
9672 // Shift Left Immediate
9673 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9674 match(Set dst (LShiftI src1 src2));
9675
9676 ins_cost(INSN_COST);
9677 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9678
9679 ins_encode %{
9680 __ lslw(as_Register($dst$$reg),
9681 as_Register($src1$$reg),
9682 $src2$$constant & 0x1f);
9683 %}
9684
9685 ins_pipe(ialu_reg_shift);
9686 %}
9687
9688 // Shift Right Logical Register
9689 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9690 match(Set dst (URShiftI src1 src2));
9691
9692 ins_cost(INSN_COST * 2);
9693 format %{ "lsrvw $dst, $src1, $src2" %}
9694
9695 ins_encode %{
9696 __ lsrvw(as_Register($dst$$reg),
9697 as_Register($src1$$reg),
9698 as_Register($src2$$reg));
9699 %}
9700
9701 ins_pipe(ialu_reg_reg_vshift);
9702 %}
9703
9704 // Shift Right Logical Immediate
9705 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9706 match(Set dst (URShiftI src1 src2));
9707
9708 ins_cost(INSN_COST);
9709 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9710
9711 ins_encode %{
9712 __ lsrw(as_Register($dst$$reg),
9713 as_Register($src1$$reg),
9714 $src2$$constant & 0x1f);
9715 %}
9716
9717 ins_pipe(ialu_reg_shift);
9718 %}
9719
9720 // Shift Right Arithmetic Register
9721 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9722 match(Set dst (RShiftI src1 src2));
9723
9724 ins_cost(INSN_COST * 2);
9725 format %{ "asrvw $dst, $src1, $src2" %}
9726
9727 ins_encode %{
9728 __ asrvw(as_Register($dst$$reg),
9729 as_Register($src1$$reg),
9730 as_Register($src2$$reg));
9731 %}
9732
9733 ins_pipe(ialu_reg_reg_vshift);
9734 %}
9735
9736 // Shift Right Arithmetic Immediate
9737 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9738 match(Set dst (RShiftI src1 src2));
9739
9740 ins_cost(INSN_COST);
9741 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9742
9743 ins_encode %{
9744 __ asrw(as_Register($dst$$reg),
9745 as_Register($src1$$reg),
9746 $src2$$constant & 0x1f);
9747 %}
9748
9749 ins_pipe(ialu_reg_shift);
9750 %}
9751
9752 // Combined Int Mask and Right Shift (using UBFM)
9753 // TODO
9754
9755 // Long Shifts
9756
9757 // Shift Left Register
9758 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9759 match(Set dst (LShiftL src1 src2));
9760
9761 ins_cost(INSN_COST * 2);
9762 format %{ "lslv $dst, $src1, $src2" %}
9763
9764 ins_encode %{
9765 __ lslv(as_Register($dst$$reg),
9766 as_Register($src1$$reg),
9767 as_Register($src2$$reg));
9768 %}
9769
9770 ins_pipe(ialu_reg_reg_vshift);
9771 %}
9772
9773 // Shift Left Immediate
9774 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9775 match(Set dst (LShiftL src1 src2));
9776
9777 ins_cost(INSN_COST);
9778 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9779
9780 ins_encode %{
9781 __ lsl(as_Register($dst$$reg),
9782 as_Register($src1$$reg),
9783 $src2$$constant & 0x3f);
9784 %}
9785
9786 ins_pipe(ialu_reg_shift);
9787 %}
9788
9789 // Shift Right Logical Register
9790 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9791 match(Set dst (URShiftL src1 src2));
9792
9793 ins_cost(INSN_COST * 2);
9794 format %{ "lsrv $dst, $src1, $src2" %}
9795
9796 ins_encode %{
9797 __ lsrv(as_Register($dst$$reg),
9798 as_Register($src1$$reg),
9799 as_Register($src2$$reg));
9800 %}
9801
9802 ins_pipe(ialu_reg_reg_vshift);
9803 %}
9804
9805 // Shift Right Logical Immediate
9806 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9807 match(Set dst (URShiftL src1 src2));
9808
9809 ins_cost(INSN_COST);
9810 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9811
9812 ins_encode %{
9813 __ lsr(as_Register($dst$$reg),
9814 as_Register($src1$$reg),
9815 $src2$$constant & 0x3f);
9816 %}
9817
9818 ins_pipe(ialu_reg_shift);
9819 %}
9820
9821 // A special-case pattern for card table stores.
9822 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9823 match(Set dst (URShiftL (CastP2X src1) src2));
9824
9825 ins_cost(INSN_COST);
9826 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9827
9828 ins_encode %{
9829 __ lsr(as_Register($dst$$reg),
9830 as_Register($src1$$reg),
9831 $src2$$constant & 0x3f);
9832 %}
9833
9834 ins_pipe(ialu_reg_shift);
9835 %}
9836
9837 // Shift Right Arithmetic Register
9838 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9839 match(Set dst (RShiftL src1 src2));
9840
9841 ins_cost(INSN_COST * 2);
9842 format %{ "asrv $dst, $src1, $src2" %}
9843
9844 ins_encode %{
9845 __ asrv(as_Register($dst$$reg),
9846 as_Register($src1$$reg),
9847 as_Register($src2$$reg));
9848 %}
9849
9850 ins_pipe(ialu_reg_reg_vshift);
9851 %}
9852
9853 // Shift Right Arithmetic Immediate
9854 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9855 match(Set dst (RShiftL src1 src2));
9856
9857 ins_cost(INSN_COST);
9858 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9859
9860 ins_encode %{
9861 __ asr(as_Register($dst$$reg),
9862 as_Register($src1$$reg),
9863 $src2$$constant & 0x3f);
9864 %}
9865
9866 ins_pipe(ialu_reg_shift);
9867 %}
9868
9869 // BEGIN This section of the file is automatically generated. Do not edit --------------
9870 // This section is generated from aarch64_ad.m4
9871
9872 // This pattern is automatically generated from aarch64_ad.m4
9873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9874 instruct regL_not_reg(iRegLNoSp dst,
9875 iRegL src1, immL_M1 m1,
9876 rFlagsReg cr) %{
9877 match(Set dst (XorL src1 m1));
9878 ins_cost(INSN_COST);
9879 format %{ "eon $dst, $src1, zr" %}
9880
9881 ins_encode %{
9882 __ eon(as_Register($dst$$reg),
9883 as_Register($src1$$reg),
9884 zr,
9885 Assembler::LSL, 0);
9886 %}
9887
9888 ins_pipe(ialu_reg);
9889 %}
9890
9891 // This pattern is automatically generated from aarch64_ad.m4
9892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9893 instruct regI_not_reg(iRegINoSp dst,
9894 iRegIorL2I src1, immI_M1 m1,
9895 rFlagsReg cr) %{
9896 match(Set dst (XorI src1 m1));
9897 ins_cost(INSN_COST);
9898 format %{ "eonw $dst, $src1, zr" %}
9899
9900 ins_encode %{
9901 __ eonw(as_Register($dst$$reg),
9902 as_Register($src1$$reg),
9903 zr,
9904 Assembler::LSL, 0);
9905 %}
9906
9907 ins_pipe(ialu_reg);
9908 %}
9909
9910 // This pattern is automatically generated from aarch64_ad.m4
9911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9912 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9913 immI0 zero, iRegIorL2I src1, immI src2) %{
9914 match(Set dst (SubI zero (URShiftI src1 src2)));
9915
9916 ins_cost(1.9 * INSN_COST);
9917 format %{ "negw $dst, $src1, LSR $src2" %}
9918
9919 ins_encode %{
9920 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9921 Assembler::LSR, $src2$$constant & 0x1f);
9922 %}
9923
9924 ins_pipe(ialu_reg_shift);
9925 %}
9926
9927 // This pattern is automatically generated from aarch64_ad.m4
9928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9929 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9930 immI0 zero, iRegIorL2I src1, immI src2) %{
9931 match(Set dst (SubI zero (RShiftI src1 src2)));
9932
9933 ins_cost(1.9 * INSN_COST);
9934 format %{ "negw $dst, $src1, ASR $src2" %}
9935
9936 ins_encode %{
9937 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9938 Assembler::ASR, $src2$$constant & 0x1f);
9939 %}
9940
9941 ins_pipe(ialu_reg_shift);
9942 %}
9943
9944 // This pattern is automatically generated from aarch64_ad.m4
9945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9946 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9947 immI0 zero, iRegIorL2I src1, immI src2) %{
9948 match(Set dst (SubI zero (LShiftI src1 src2)));
9949
9950 ins_cost(1.9 * INSN_COST);
9951 format %{ "negw $dst, $src1, LSL $src2" %}
9952
9953 ins_encode %{
9954 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9955 Assembler::LSL, $src2$$constant & 0x1f);
9956 %}
9957
9958 ins_pipe(ialu_reg_shift);
9959 %}
9960
9961 // This pattern is automatically generated from aarch64_ad.m4
9962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9963 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9964 immL0 zero, iRegL src1, immI src2) %{
9965 match(Set dst (SubL zero (URShiftL src1 src2)));
9966
9967 ins_cost(1.9 * INSN_COST);
9968 format %{ "neg $dst, $src1, LSR $src2" %}
9969
9970 ins_encode %{
9971 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9972 Assembler::LSR, $src2$$constant & 0x3f);
9973 %}
9974
9975 ins_pipe(ialu_reg_shift);
9976 %}
9977
9978 // This pattern is automatically generated from aarch64_ad.m4
9979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9980 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9981 immL0 zero, iRegL src1, immI src2) %{
9982 match(Set dst (SubL zero (RShiftL src1 src2)));
9983
9984 ins_cost(1.9 * INSN_COST);
9985 format %{ "neg $dst, $src1, ASR $src2" %}
9986
9987 ins_encode %{
9988 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9989 Assembler::ASR, $src2$$constant & 0x3f);
9990 %}
9991
9992 ins_pipe(ialu_reg_shift);
9993 %}
9994
9995 // This pattern is automatically generated from aarch64_ad.m4
9996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9997 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
9998 immL0 zero, iRegL src1, immI src2) %{
9999 match(Set dst (SubL zero (LShiftL src1 src2)));
10000
10001 ins_cost(1.9 * INSN_COST);
10002 format %{ "neg $dst, $src1, LSL $src2" %}
10003
10004 ins_encode %{
10005 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10006 Assembler::LSL, $src2$$constant & 0x3f);
10007 %}
10008
10009 ins_pipe(ialu_reg_shift);
10010 %}
10011
10012 // This pattern is automatically generated from aarch64_ad.m4
10013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10014 instruct AndI_reg_not_reg(iRegINoSp dst,
10015 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10016 match(Set dst (AndI src1 (XorI src2 m1)));
10017 ins_cost(INSN_COST);
10018 format %{ "bicw $dst, $src1, $src2" %}
10019
10020 ins_encode %{
10021 __ bicw(as_Register($dst$$reg),
10022 as_Register($src1$$reg),
10023 as_Register($src2$$reg),
10024 Assembler::LSL, 0);
10025 %}
10026
10027 ins_pipe(ialu_reg_reg);
10028 %}
10029
10030 // This pattern is automatically generated from aarch64_ad.m4
10031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10032 instruct AndL_reg_not_reg(iRegLNoSp dst,
10033 iRegL src1, iRegL src2, immL_M1 m1) %{
10034 match(Set dst (AndL src1 (XorL src2 m1)));
10035 ins_cost(INSN_COST);
10036 format %{ "bic $dst, $src1, $src2" %}
10037
10038 ins_encode %{
10039 __ bic(as_Register($dst$$reg),
10040 as_Register($src1$$reg),
10041 as_Register($src2$$reg),
10042 Assembler::LSL, 0);
10043 %}
10044
10045 ins_pipe(ialu_reg_reg);
10046 %}
10047
10048 // This pattern is automatically generated from aarch64_ad.m4
10049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10050 instruct OrI_reg_not_reg(iRegINoSp dst,
10051 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10052 match(Set dst (OrI src1 (XorI src2 m1)));
10053 ins_cost(INSN_COST);
10054 format %{ "ornw $dst, $src1, $src2" %}
10055
10056 ins_encode %{
10057 __ ornw(as_Register($dst$$reg),
10058 as_Register($src1$$reg),
10059 as_Register($src2$$reg),
10060 Assembler::LSL, 0);
10061 %}
10062
10063 ins_pipe(ialu_reg_reg);
10064 %}
10065
10066 // This pattern is automatically generated from aarch64_ad.m4
10067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10068 instruct OrL_reg_not_reg(iRegLNoSp dst,
10069 iRegL src1, iRegL src2, immL_M1 m1) %{
10070 match(Set dst (OrL src1 (XorL src2 m1)));
10071 ins_cost(INSN_COST);
10072 format %{ "orn $dst, $src1, $src2" %}
10073
10074 ins_encode %{
10075 __ orn(as_Register($dst$$reg),
10076 as_Register($src1$$reg),
10077 as_Register($src2$$reg),
10078 Assembler::LSL, 0);
10079 %}
10080
10081 ins_pipe(ialu_reg_reg);
10082 %}
10083
10084 // This pattern is automatically generated from aarch64_ad.m4
10085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10086 instruct XorI_reg_not_reg(iRegINoSp dst,
10087 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10088 match(Set dst (XorI m1 (XorI src2 src1)));
10089 ins_cost(INSN_COST);
10090 format %{ "eonw $dst, $src1, $src2" %}
10091
10092 ins_encode %{
10093 __ eonw(as_Register($dst$$reg),
10094 as_Register($src1$$reg),
10095 as_Register($src2$$reg),
10096 Assembler::LSL, 0);
10097 %}
10098
10099 ins_pipe(ialu_reg_reg);
10100 %}
10101
10102 // This pattern is automatically generated from aarch64_ad.m4
10103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10104 instruct XorL_reg_not_reg(iRegLNoSp dst,
10105 iRegL src1, iRegL src2, immL_M1 m1) %{
10106 match(Set dst (XorL m1 (XorL src2 src1)));
10107 ins_cost(INSN_COST);
10108 format %{ "eon $dst, $src1, $src2" %}
10109
10110 ins_encode %{
10111 __ eon(as_Register($dst$$reg),
10112 as_Register($src1$$reg),
10113 as_Register($src2$$reg),
10114 Assembler::LSL, 0);
10115 %}
10116
10117 ins_pipe(ialu_reg_reg);
10118 %}
10119
10120 // This pattern is automatically generated from aarch64_ad.m4
10121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10122 // val & (-1 ^ (val >>> shift)) ==> bicw
10123 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10124 iRegIorL2I src1, iRegIorL2I src2,
10125 immI src3, immI_M1 src4) %{
10126 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10127 ins_cost(1.9 * INSN_COST);
10128 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10129
10130 ins_encode %{
10131 __ bicw(as_Register($dst$$reg),
10132 as_Register($src1$$reg),
10133 as_Register($src2$$reg),
10134 Assembler::LSR,
10135 $src3$$constant & 0x1f);
10136 %}
10137
10138 ins_pipe(ialu_reg_reg_shift);
10139 %}
10140
10141 // This pattern is automatically generated from aarch64_ad.m4
10142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10143 // val & (-1 ^ (val >>> shift)) ==> bic
10144 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10145 iRegL src1, iRegL src2,
10146 immI src3, immL_M1 src4) %{
10147 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10148 ins_cost(1.9 * INSN_COST);
10149 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10150
10151 ins_encode %{
10152 __ bic(as_Register($dst$$reg),
10153 as_Register($src1$$reg),
10154 as_Register($src2$$reg),
10155 Assembler::LSR,
10156 $src3$$constant & 0x3f);
10157 %}
10158
10159 ins_pipe(ialu_reg_reg_shift);
10160 %}
10161
10162 // This pattern is automatically generated from aarch64_ad.m4
10163 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10164 // val & (-1 ^ (val >> shift)) ==> bicw
10165 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10166 iRegIorL2I src1, iRegIorL2I src2,
10167 immI src3, immI_M1 src4) %{
10168 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10169 ins_cost(1.9 * INSN_COST);
10170 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10171
10172 ins_encode %{
10173 __ bicw(as_Register($dst$$reg),
10174 as_Register($src1$$reg),
10175 as_Register($src2$$reg),
10176 Assembler::ASR,
10177 $src3$$constant & 0x1f);
10178 %}
10179
10180 ins_pipe(ialu_reg_reg_shift);
10181 %}
10182
10183 // This pattern is automatically generated from aarch64_ad.m4
10184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10185 // val & (-1 ^ (val >> shift)) ==> bic
10186 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10187 iRegL src1, iRegL src2,
10188 immI src3, immL_M1 src4) %{
10189 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10190 ins_cost(1.9 * INSN_COST);
10191 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10192
10193 ins_encode %{
10194 __ bic(as_Register($dst$$reg),
10195 as_Register($src1$$reg),
10196 as_Register($src2$$reg),
10197 Assembler::ASR,
10198 $src3$$constant & 0x3f);
10199 %}
10200
10201 ins_pipe(ialu_reg_reg_shift);
10202 %}
10203
10204 // This pattern is automatically generated from aarch64_ad.m4
10205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10206 // val & (-1 ^ (val ror shift)) ==> bicw
10207 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10208 iRegIorL2I src1, iRegIorL2I src2,
10209 immI src3, immI_M1 src4) %{
10210 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10211 ins_cost(1.9 * INSN_COST);
10212 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10213
10214 ins_encode %{
10215 __ bicw(as_Register($dst$$reg),
10216 as_Register($src1$$reg),
10217 as_Register($src2$$reg),
10218 Assembler::ROR,
10219 $src3$$constant & 0x1f);
10220 %}
10221
10222 ins_pipe(ialu_reg_reg_shift);
10223 %}
10224
10225 // This pattern is automatically generated from aarch64_ad.m4
10226 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10227 // val & (-1 ^ (val ror shift)) ==> bic
10228 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10229 iRegL src1, iRegL src2,
10230 immI src3, immL_M1 src4) %{
10231 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10232 ins_cost(1.9 * INSN_COST);
10233 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10234
10235 ins_encode %{
10236 __ bic(as_Register($dst$$reg),
10237 as_Register($src1$$reg),
10238 as_Register($src2$$reg),
10239 Assembler::ROR,
10240 $src3$$constant & 0x3f);
10241 %}
10242
10243 ins_pipe(ialu_reg_reg_shift);
10244 %}
10245
10246 // This pattern is automatically generated from aarch64_ad.m4
10247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10248 // val & (-1 ^ (val << shift)) ==> bicw
10249 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10250 iRegIorL2I src1, iRegIorL2I src2,
10251 immI src3, immI_M1 src4) %{
10252 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10253 ins_cost(1.9 * INSN_COST);
10254 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10255
10256 ins_encode %{
10257 __ bicw(as_Register($dst$$reg),
10258 as_Register($src1$$reg),
10259 as_Register($src2$$reg),
10260 Assembler::LSL,
10261 $src3$$constant & 0x1f);
10262 %}
10263
10264 ins_pipe(ialu_reg_reg_shift);
10265 %}
10266
10267 // This pattern is automatically generated from aarch64_ad.m4
10268 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10269 // val & (-1 ^ (val << shift)) ==> bic
10270 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10271 iRegL src1, iRegL src2,
10272 immI src3, immL_M1 src4) %{
10273 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10274 ins_cost(1.9 * INSN_COST);
10275 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10276
10277 ins_encode %{
10278 __ bic(as_Register($dst$$reg),
10279 as_Register($src1$$reg),
10280 as_Register($src2$$reg),
10281 Assembler::LSL,
10282 $src3$$constant & 0x3f);
10283 %}
10284
10285 ins_pipe(ialu_reg_reg_shift);
10286 %}
10287
10288 // This pattern is automatically generated from aarch64_ad.m4
10289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10290 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10291 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10292 iRegIorL2I src1, iRegIorL2I src2,
10293 immI src3, immI_M1 src4) %{
10294 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10295 ins_cost(1.9 * INSN_COST);
10296 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10297
10298 ins_encode %{
10299 __ eonw(as_Register($dst$$reg),
10300 as_Register($src1$$reg),
10301 as_Register($src2$$reg),
10302 Assembler::LSR,
10303 $src3$$constant & 0x1f);
10304 %}
10305
10306 ins_pipe(ialu_reg_reg_shift);
10307 %}
10308
10309 // This pattern is automatically generated from aarch64_ad.m4
10310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10311 // val ^ (-1 ^ (val >>> shift)) ==> eon
10312 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10313 iRegL src1, iRegL src2,
10314 immI src3, immL_M1 src4) %{
10315 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10316 ins_cost(1.9 * INSN_COST);
10317 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10318
10319 ins_encode %{
10320 __ eon(as_Register($dst$$reg),
10321 as_Register($src1$$reg),
10322 as_Register($src2$$reg),
10323 Assembler::LSR,
10324 $src3$$constant & 0x3f);
10325 %}
10326
10327 ins_pipe(ialu_reg_reg_shift);
10328 %}
10329
10330 // This pattern is automatically generated from aarch64_ad.m4
10331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10332 // val ^ (-1 ^ (val >> shift)) ==> eonw
10333 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10334 iRegIorL2I src1, iRegIorL2I src2,
10335 immI src3, immI_M1 src4) %{
10336 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10337 ins_cost(1.9 * INSN_COST);
10338 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10339
10340 ins_encode %{
10341 __ eonw(as_Register($dst$$reg),
10342 as_Register($src1$$reg),
10343 as_Register($src2$$reg),
10344 Assembler::ASR,
10345 $src3$$constant & 0x1f);
10346 %}
10347
10348 ins_pipe(ialu_reg_reg_shift);
10349 %}
10350
10351 // This pattern is automatically generated from aarch64_ad.m4
10352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10353 // val ^ (-1 ^ (val >> shift)) ==> eon
10354 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10355 iRegL src1, iRegL src2,
10356 immI src3, immL_M1 src4) %{
10357 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10358 ins_cost(1.9 * INSN_COST);
10359 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10360
10361 ins_encode %{
10362 __ eon(as_Register($dst$$reg),
10363 as_Register($src1$$reg),
10364 as_Register($src2$$reg),
10365 Assembler::ASR,
10366 $src3$$constant & 0x3f);
10367 %}
10368
10369 ins_pipe(ialu_reg_reg_shift);
10370 %}
10371
10372 // This pattern is automatically generated from aarch64_ad.m4
10373 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10374 // val ^ (-1 ^ (val ror shift)) ==> eonw
10375 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10376 iRegIorL2I src1, iRegIorL2I src2,
10377 immI src3, immI_M1 src4) %{
10378 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10379 ins_cost(1.9 * INSN_COST);
10380 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10381
10382 ins_encode %{
10383 __ eonw(as_Register($dst$$reg),
10384 as_Register($src1$$reg),
10385 as_Register($src2$$reg),
10386 Assembler::ROR,
10387 $src3$$constant & 0x1f);
10388 %}
10389
10390 ins_pipe(ialu_reg_reg_shift);
10391 %}
10392
10393 // This pattern is automatically generated from aarch64_ad.m4
10394 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10395 // val ^ (-1 ^ (val ror shift)) ==> eon
10396 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10397 iRegL src1, iRegL src2,
10398 immI src3, immL_M1 src4) %{
10399 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10400 ins_cost(1.9 * INSN_COST);
10401 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10402
10403 ins_encode %{
10404 __ eon(as_Register($dst$$reg),
10405 as_Register($src1$$reg),
10406 as_Register($src2$$reg),
10407 Assembler::ROR,
10408 $src3$$constant & 0x3f);
10409 %}
10410
10411 ins_pipe(ialu_reg_reg_shift);
10412 %}
10413
10414 // This pattern is automatically generated from aarch64_ad.m4
10415 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10416 // val ^ (-1 ^ (val << shift)) ==> eonw
10417 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10418 iRegIorL2I src1, iRegIorL2I src2,
10419 immI src3, immI_M1 src4) %{
10420 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10421 ins_cost(1.9 * INSN_COST);
10422 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10423
10424 ins_encode %{
10425 __ eonw(as_Register($dst$$reg),
10426 as_Register($src1$$reg),
10427 as_Register($src2$$reg),
10428 Assembler::LSL,
10429 $src3$$constant & 0x1f);
10430 %}
10431
10432 ins_pipe(ialu_reg_reg_shift);
10433 %}
10434
10435 // This pattern is automatically generated from aarch64_ad.m4
10436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10437 // val ^ (-1 ^ (val << shift)) ==> eon
10438 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10439 iRegL src1, iRegL src2,
10440 immI src3, immL_M1 src4) %{
10441 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10442 ins_cost(1.9 * INSN_COST);
10443 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10444
10445 ins_encode %{
10446 __ eon(as_Register($dst$$reg),
10447 as_Register($src1$$reg),
10448 as_Register($src2$$reg),
10449 Assembler::LSL,
10450 $src3$$constant & 0x3f);
10451 %}
10452
10453 ins_pipe(ialu_reg_reg_shift);
10454 %}
10455
10456 // This pattern is automatically generated from aarch64_ad.m4
10457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10458 // val | (-1 ^ (val >>> shift)) ==> ornw
10459 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10460 iRegIorL2I src1, iRegIorL2I src2,
10461 immI src3, immI_M1 src4) %{
10462 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10463 ins_cost(1.9 * INSN_COST);
10464 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10465
10466 ins_encode %{
10467 __ ornw(as_Register($dst$$reg),
10468 as_Register($src1$$reg),
10469 as_Register($src2$$reg),
10470 Assembler::LSR,
10471 $src3$$constant & 0x1f);
10472 %}
10473
10474 ins_pipe(ialu_reg_reg_shift);
10475 %}
10476
10477 // This pattern is automatically generated from aarch64_ad.m4
10478 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10479 // val | (-1 ^ (val >>> shift)) ==> orn
10480 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10481 iRegL src1, iRegL src2,
10482 immI src3, immL_M1 src4) %{
10483 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10484 ins_cost(1.9 * INSN_COST);
10485 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10486
10487 ins_encode %{
10488 __ orn(as_Register($dst$$reg),
10489 as_Register($src1$$reg),
10490 as_Register($src2$$reg),
10491 Assembler::LSR,
10492 $src3$$constant & 0x3f);
10493 %}
10494
10495 ins_pipe(ialu_reg_reg_shift);
10496 %}
10497
10498 // This pattern is automatically generated from aarch64_ad.m4
10499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10500 // val | (-1 ^ (val >> shift)) ==> ornw
10501 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10502 iRegIorL2I src1, iRegIorL2I src2,
10503 immI src3, immI_M1 src4) %{
10504 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10505 ins_cost(1.9 * INSN_COST);
10506 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10507
10508 ins_encode %{
10509 __ ornw(as_Register($dst$$reg),
10510 as_Register($src1$$reg),
10511 as_Register($src2$$reg),
10512 Assembler::ASR,
10513 $src3$$constant & 0x1f);
10514 %}
10515
10516 ins_pipe(ialu_reg_reg_shift);
10517 %}
10518
10519 // This pattern is automatically generated from aarch64_ad.m4
10520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10521 // val | (-1 ^ (val >> shift)) ==> orn
10522 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10523 iRegL src1, iRegL src2,
10524 immI src3, immL_M1 src4) %{
10525 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10526 ins_cost(1.9 * INSN_COST);
10527 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10528
10529 ins_encode %{
10530 __ orn(as_Register($dst$$reg),
10531 as_Register($src1$$reg),
10532 as_Register($src2$$reg),
10533 Assembler::ASR,
10534 $src3$$constant & 0x3f);
10535 %}
10536
10537 ins_pipe(ialu_reg_reg_shift);
10538 %}
10539
10540 // This pattern is automatically generated from aarch64_ad.m4
10541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10542 // val | (-1 ^ (val ror shift)) ==> ornw
10543 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10544 iRegIorL2I src1, iRegIorL2I src2,
10545 immI src3, immI_M1 src4) %{
10546 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10547 ins_cost(1.9 * INSN_COST);
10548 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10549
10550 ins_encode %{
10551 __ ornw(as_Register($dst$$reg),
10552 as_Register($src1$$reg),
10553 as_Register($src2$$reg),
10554 Assembler::ROR,
10555 $src3$$constant & 0x1f);
10556 %}
10557
10558 ins_pipe(ialu_reg_reg_shift);
10559 %}
10560
10561 // This pattern is automatically generated from aarch64_ad.m4
10562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10563 // val | (-1 ^ (val ror shift)) ==> orn
10564 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10565 iRegL src1, iRegL src2,
10566 immI src3, immL_M1 src4) %{
10567 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10568 ins_cost(1.9 * INSN_COST);
10569 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10570
10571 ins_encode %{
10572 __ orn(as_Register($dst$$reg),
10573 as_Register($src1$$reg),
10574 as_Register($src2$$reg),
10575 Assembler::ROR,
10576 $src3$$constant & 0x3f);
10577 %}
10578
10579 ins_pipe(ialu_reg_reg_shift);
10580 %}
10581
10582 // This pattern is automatically generated from aarch64_ad.m4
10583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10584 // val | (-1 ^ (val << shift)) ==> ornw
10585 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10586 iRegIorL2I src1, iRegIorL2I src2,
10587 immI src3, immI_M1 src4) %{
10588 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10589 ins_cost(1.9 * INSN_COST);
10590 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10591
10592 ins_encode %{
10593 __ ornw(as_Register($dst$$reg),
10594 as_Register($src1$$reg),
10595 as_Register($src2$$reg),
10596 Assembler::LSL,
10597 $src3$$constant & 0x1f);
10598 %}
10599
10600 ins_pipe(ialu_reg_reg_shift);
10601 %}
10602
10603 // This pattern is automatically generated from aarch64_ad.m4
10604 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10605 // val | (-1 ^ (val << shift)) ==> orn
10606 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10607 iRegL src1, iRegL src2,
10608 immI src3, immL_M1 src4) %{
10609 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10610 ins_cost(1.9 * INSN_COST);
10611 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10612
10613 ins_encode %{
10614 __ orn(as_Register($dst$$reg),
10615 as_Register($src1$$reg),
10616 as_Register($src2$$reg),
10617 Assembler::LSL,
10618 $src3$$constant & 0x3f);
10619 %}
10620
10621 ins_pipe(ialu_reg_reg_shift);
10622 %}
10623
10624 // This pattern is automatically generated from aarch64_ad.m4
10625 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10626 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10627 iRegIorL2I src1, iRegIorL2I src2,
10628 immI src3) %{
10629 match(Set dst (AndI src1 (URShiftI src2 src3)));
10630
10631 ins_cost(1.9 * INSN_COST);
10632 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10633
10634 ins_encode %{
10635 __ andw(as_Register($dst$$reg),
10636 as_Register($src1$$reg),
10637 as_Register($src2$$reg),
10638 Assembler::LSR,
10639 $src3$$constant & 0x1f);
10640 %}
10641
10642 ins_pipe(ialu_reg_reg_shift);
10643 %}
10644
10645 // This pattern is automatically generated from aarch64_ad.m4
10646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10647 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10648 iRegL src1, iRegL src2,
10649 immI src3) %{
10650 match(Set dst (AndL src1 (URShiftL src2 src3)));
10651
10652 ins_cost(1.9 * INSN_COST);
10653 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10654
10655 ins_encode %{
10656 __ andr(as_Register($dst$$reg),
10657 as_Register($src1$$reg),
10658 as_Register($src2$$reg),
10659 Assembler::LSR,
10660 $src3$$constant & 0x3f);
10661 %}
10662
10663 ins_pipe(ialu_reg_reg_shift);
10664 %}
10665
10666 // This pattern is automatically generated from aarch64_ad.m4
10667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10668 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10669 iRegIorL2I src1, iRegIorL2I src2,
10670 immI src3) %{
10671 match(Set dst (AndI src1 (RShiftI src2 src3)));
10672
10673 ins_cost(1.9 * INSN_COST);
10674 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10675
10676 ins_encode %{
10677 __ andw(as_Register($dst$$reg),
10678 as_Register($src1$$reg),
10679 as_Register($src2$$reg),
10680 Assembler::ASR,
10681 $src3$$constant & 0x1f);
10682 %}
10683
10684 ins_pipe(ialu_reg_reg_shift);
10685 %}
10686
10687 // This pattern is automatically generated from aarch64_ad.m4
10688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10689 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10690 iRegL src1, iRegL src2,
10691 immI src3) %{
10692 match(Set dst (AndL src1 (RShiftL src2 src3)));
10693
10694 ins_cost(1.9 * INSN_COST);
10695 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10696
10697 ins_encode %{
10698 __ andr(as_Register($dst$$reg),
10699 as_Register($src1$$reg),
10700 as_Register($src2$$reg),
10701 Assembler::ASR,
10702 $src3$$constant & 0x3f);
10703 %}
10704
10705 ins_pipe(ialu_reg_reg_shift);
10706 %}
10707
10708 // This pattern is automatically generated from aarch64_ad.m4
10709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10710 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10711 iRegIorL2I src1, iRegIorL2I src2,
10712 immI src3) %{
10713 match(Set dst (AndI src1 (LShiftI src2 src3)));
10714
10715 ins_cost(1.9 * INSN_COST);
10716 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10717
10718 ins_encode %{
10719 __ andw(as_Register($dst$$reg),
10720 as_Register($src1$$reg),
10721 as_Register($src2$$reg),
10722 Assembler::LSL,
10723 $src3$$constant & 0x1f);
10724 %}
10725
10726 ins_pipe(ialu_reg_reg_shift);
10727 %}
10728
10729 // This pattern is automatically generated from aarch64_ad.m4
10730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10731 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10732 iRegL src1, iRegL src2,
10733 immI src3) %{
10734 match(Set dst (AndL src1 (LShiftL src2 src3)));
10735
10736 ins_cost(1.9 * INSN_COST);
10737 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10738
10739 ins_encode %{
10740 __ andr(as_Register($dst$$reg),
10741 as_Register($src1$$reg),
10742 as_Register($src2$$reg),
10743 Assembler::LSL,
10744 $src3$$constant & 0x3f);
10745 %}
10746
10747 ins_pipe(ialu_reg_reg_shift);
10748 %}
10749
10750 // This pattern is automatically generated from aarch64_ad.m4
10751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10752 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10753 iRegIorL2I src1, iRegIorL2I src2,
10754 immI src3) %{
10755 match(Set dst (AndI src1 (RotateRight src2 src3)));
10756
10757 ins_cost(1.9 * INSN_COST);
10758 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10759
10760 ins_encode %{
10761 __ andw(as_Register($dst$$reg),
10762 as_Register($src1$$reg),
10763 as_Register($src2$$reg),
10764 Assembler::ROR,
10765 $src3$$constant & 0x1f);
10766 %}
10767
10768 ins_pipe(ialu_reg_reg_shift);
10769 %}
10770
10771 // This pattern is automatically generated from aarch64_ad.m4
10772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10773 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10774 iRegL src1, iRegL src2,
10775 immI src3) %{
10776 match(Set dst (AndL src1 (RotateRight src2 src3)));
10777
10778 ins_cost(1.9 * INSN_COST);
10779 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10780
10781 ins_encode %{
10782 __ andr(as_Register($dst$$reg),
10783 as_Register($src1$$reg),
10784 as_Register($src2$$reg),
10785 Assembler::ROR,
10786 $src3$$constant & 0x3f);
10787 %}
10788
10789 ins_pipe(ialu_reg_reg_shift);
10790 %}
10791
10792 // This pattern is automatically generated from aarch64_ad.m4
10793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10794 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10795 iRegIorL2I src1, iRegIorL2I src2,
10796 immI src3) %{
10797 match(Set dst (XorI src1 (URShiftI src2 src3)));
10798
10799 ins_cost(1.9 * INSN_COST);
10800 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10801
10802 ins_encode %{
10803 __ eorw(as_Register($dst$$reg),
10804 as_Register($src1$$reg),
10805 as_Register($src2$$reg),
10806 Assembler::LSR,
10807 $src3$$constant & 0x1f);
10808 %}
10809
10810 ins_pipe(ialu_reg_reg_shift);
10811 %}
10812
10813 // This pattern is automatically generated from aarch64_ad.m4
10814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10815 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10816 iRegL src1, iRegL src2,
10817 immI src3) %{
10818 match(Set dst (XorL src1 (URShiftL src2 src3)));
10819
10820 ins_cost(1.9 * INSN_COST);
10821 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10822
10823 ins_encode %{
10824 __ eor(as_Register($dst$$reg),
10825 as_Register($src1$$reg),
10826 as_Register($src2$$reg),
10827 Assembler::LSR,
10828 $src3$$constant & 0x3f);
10829 %}
10830
10831 ins_pipe(ialu_reg_reg_shift);
10832 %}
10833
10834 // This pattern is automatically generated from aarch64_ad.m4
10835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10836 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10837 iRegIorL2I src1, iRegIorL2I src2,
10838 immI src3) %{
10839 match(Set dst (XorI src1 (RShiftI src2 src3)));
10840
10841 ins_cost(1.9 * INSN_COST);
10842 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10843
10844 ins_encode %{
10845 __ eorw(as_Register($dst$$reg),
10846 as_Register($src1$$reg),
10847 as_Register($src2$$reg),
10848 Assembler::ASR,
10849 $src3$$constant & 0x1f);
10850 %}
10851
10852 ins_pipe(ialu_reg_reg_shift);
10853 %}
10854
10855 // This pattern is automatically generated from aarch64_ad.m4
10856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10857 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10858 iRegL src1, iRegL src2,
10859 immI src3) %{
10860 match(Set dst (XorL src1 (RShiftL src2 src3)));
10861
10862 ins_cost(1.9 * INSN_COST);
10863 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10864
10865 ins_encode %{
10866 __ eor(as_Register($dst$$reg),
10867 as_Register($src1$$reg),
10868 as_Register($src2$$reg),
10869 Assembler::ASR,
10870 $src3$$constant & 0x3f);
10871 %}
10872
10873 ins_pipe(ialu_reg_reg_shift);
10874 %}
10875
10876 // This pattern is automatically generated from aarch64_ad.m4
10877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10878 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10879 iRegIorL2I src1, iRegIorL2I src2,
10880 immI src3) %{
10881 match(Set dst (XorI src1 (LShiftI src2 src3)));
10882
10883 ins_cost(1.9 * INSN_COST);
10884 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10885
10886 ins_encode %{
10887 __ eorw(as_Register($dst$$reg),
10888 as_Register($src1$$reg),
10889 as_Register($src2$$reg),
10890 Assembler::LSL,
10891 $src3$$constant & 0x1f);
10892 %}
10893
10894 ins_pipe(ialu_reg_reg_shift);
10895 %}
10896
10897 // This pattern is automatically generated from aarch64_ad.m4
10898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10899 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10900 iRegL src1, iRegL src2,
10901 immI src3) %{
10902 match(Set dst (XorL src1 (LShiftL src2 src3)));
10903
10904 ins_cost(1.9 * INSN_COST);
10905 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10906
10907 ins_encode %{
10908 __ eor(as_Register($dst$$reg),
10909 as_Register($src1$$reg),
10910 as_Register($src2$$reg),
10911 Assembler::LSL,
10912 $src3$$constant & 0x3f);
10913 %}
10914
10915 ins_pipe(ialu_reg_reg_shift);
10916 %}
10917
10918 // This pattern is automatically generated from aarch64_ad.m4
10919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10920 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10921 iRegIorL2I src1, iRegIorL2I src2,
10922 immI src3) %{
10923 match(Set dst (XorI src1 (RotateRight src2 src3)));
10924
10925 ins_cost(1.9 * INSN_COST);
10926 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10927
10928 ins_encode %{
10929 __ eorw(as_Register($dst$$reg),
10930 as_Register($src1$$reg),
10931 as_Register($src2$$reg),
10932 Assembler::ROR,
10933 $src3$$constant & 0x1f);
10934 %}
10935
10936 ins_pipe(ialu_reg_reg_shift);
10937 %}
10938
10939 // This pattern is automatically generated from aarch64_ad.m4
10940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10941 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10942 iRegL src1, iRegL src2,
10943 immI src3) %{
10944 match(Set dst (XorL src1 (RotateRight src2 src3)));
10945
10946 ins_cost(1.9 * INSN_COST);
10947 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10948
10949 ins_encode %{
10950 __ eor(as_Register($dst$$reg),
10951 as_Register($src1$$reg),
10952 as_Register($src2$$reg),
10953 Assembler::ROR,
10954 $src3$$constant & 0x3f);
10955 %}
10956
10957 ins_pipe(ialu_reg_reg_shift);
10958 %}
10959
10960 // This pattern is automatically generated from aarch64_ad.m4
10961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10962 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10963 iRegIorL2I src1, iRegIorL2I src2,
10964 immI src3) %{
10965 match(Set dst (OrI src1 (URShiftI src2 src3)));
10966
10967 ins_cost(1.9 * INSN_COST);
10968 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10969
10970 ins_encode %{
10971 __ orrw(as_Register($dst$$reg),
10972 as_Register($src1$$reg),
10973 as_Register($src2$$reg),
10974 Assembler::LSR,
10975 $src3$$constant & 0x1f);
10976 %}
10977
10978 ins_pipe(ialu_reg_reg_shift);
10979 %}
10980
10981 // This pattern is automatically generated from aarch64_ad.m4
10982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10983 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10984 iRegL src1, iRegL src2,
10985 immI src3) %{
10986 match(Set dst (OrL src1 (URShiftL src2 src3)));
10987
10988 ins_cost(1.9 * INSN_COST);
10989 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10990
10991 ins_encode %{
10992 __ orr(as_Register($dst$$reg),
10993 as_Register($src1$$reg),
10994 as_Register($src2$$reg),
10995 Assembler::LSR,
10996 $src3$$constant & 0x3f);
10997 %}
10998
10999 ins_pipe(ialu_reg_reg_shift);
11000 %}
11001
11002 // This pattern is automatically generated from aarch64_ad.m4
11003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11004 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11005 iRegIorL2I src1, iRegIorL2I src2,
11006 immI src3) %{
11007 match(Set dst (OrI src1 (RShiftI src2 src3)));
11008
11009 ins_cost(1.9 * INSN_COST);
11010 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11011
11012 ins_encode %{
11013 __ orrw(as_Register($dst$$reg),
11014 as_Register($src1$$reg),
11015 as_Register($src2$$reg),
11016 Assembler::ASR,
11017 $src3$$constant & 0x1f);
11018 %}
11019
11020 ins_pipe(ialu_reg_reg_shift);
11021 %}
11022
11023 // This pattern is automatically generated from aarch64_ad.m4
11024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11025 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11026 iRegL src1, iRegL src2,
11027 immI src3) %{
11028 match(Set dst (OrL src1 (RShiftL src2 src3)));
11029
11030 ins_cost(1.9 * INSN_COST);
11031 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11032
11033 ins_encode %{
11034 __ orr(as_Register($dst$$reg),
11035 as_Register($src1$$reg),
11036 as_Register($src2$$reg),
11037 Assembler::ASR,
11038 $src3$$constant & 0x3f);
11039 %}
11040
11041 ins_pipe(ialu_reg_reg_shift);
11042 %}
11043
11044 // This pattern is automatically generated from aarch64_ad.m4
11045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11046 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11047 iRegIorL2I src1, iRegIorL2I src2,
11048 immI src3) %{
11049 match(Set dst (OrI src1 (LShiftI src2 src3)));
11050
11051 ins_cost(1.9 * INSN_COST);
11052 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11053
11054 ins_encode %{
11055 __ orrw(as_Register($dst$$reg),
11056 as_Register($src1$$reg),
11057 as_Register($src2$$reg),
11058 Assembler::LSL,
11059 $src3$$constant & 0x1f);
11060 %}
11061
11062 ins_pipe(ialu_reg_reg_shift);
11063 %}
11064
11065 // This pattern is automatically generated from aarch64_ad.m4
11066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11067 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11068 iRegL src1, iRegL src2,
11069 immI src3) %{
11070 match(Set dst (OrL src1 (LShiftL src2 src3)));
11071
11072 ins_cost(1.9 * INSN_COST);
11073 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11074
11075 ins_encode %{
11076 __ orr(as_Register($dst$$reg),
11077 as_Register($src1$$reg),
11078 as_Register($src2$$reg),
11079 Assembler::LSL,
11080 $src3$$constant & 0x3f);
11081 %}
11082
11083 ins_pipe(ialu_reg_reg_shift);
11084 %}
11085
11086 // This pattern is automatically generated from aarch64_ad.m4
11087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11088 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11089 iRegIorL2I src1, iRegIorL2I src2,
11090 immI src3) %{
11091 match(Set dst (OrI src1 (RotateRight src2 src3)));
11092
11093 ins_cost(1.9 * INSN_COST);
11094 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11095
11096 ins_encode %{
11097 __ orrw(as_Register($dst$$reg),
11098 as_Register($src1$$reg),
11099 as_Register($src2$$reg),
11100 Assembler::ROR,
11101 $src3$$constant & 0x1f);
11102 %}
11103
11104 ins_pipe(ialu_reg_reg_shift);
11105 %}
11106
11107 // This pattern is automatically generated from aarch64_ad.m4
11108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11109 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11110 iRegL src1, iRegL src2,
11111 immI src3) %{
11112 match(Set dst (OrL src1 (RotateRight src2 src3)));
11113
11114 ins_cost(1.9 * INSN_COST);
11115 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11116
11117 ins_encode %{
11118 __ orr(as_Register($dst$$reg),
11119 as_Register($src1$$reg),
11120 as_Register($src2$$reg),
11121 Assembler::ROR,
11122 $src3$$constant & 0x3f);
11123 %}
11124
11125 ins_pipe(ialu_reg_reg_shift);
11126 %}
11127
11128 // This pattern is automatically generated from aarch64_ad.m4
11129 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11130 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11131 iRegIorL2I src1, iRegIorL2I src2,
11132 immI src3) %{
11133 match(Set dst (AddI src1 (URShiftI src2 src3)));
11134
11135 ins_cost(1.9 * INSN_COST);
11136 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11137
11138 ins_encode %{
11139 __ addw(as_Register($dst$$reg),
11140 as_Register($src1$$reg),
11141 as_Register($src2$$reg),
11142 Assembler::LSR,
11143 $src3$$constant & 0x1f);
11144 %}
11145
11146 ins_pipe(ialu_reg_reg_shift);
11147 %}
11148
11149 // This pattern is automatically generated from aarch64_ad.m4
11150 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11151 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11152 iRegL src1, iRegL src2,
11153 immI src3) %{
11154 match(Set dst (AddL src1 (URShiftL src2 src3)));
11155
11156 ins_cost(1.9 * INSN_COST);
11157 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11158
11159 ins_encode %{
11160 __ add(as_Register($dst$$reg),
11161 as_Register($src1$$reg),
11162 as_Register($src2$$reg),
11163 Assembler::LSR,
11164 $src3$$constant & 0x3f);
11165 %}
11166
11167 ins_pipe(ialu_reg_reg_shift);
11168 %}
11169
11170 // This pattern is automatically generated from aarch64_ad.m4
11171 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11172 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11173 iRegIorL2I src1, iRegIorL2I src2,
11174 immI src3) %{
11175 match(Set dst (AddI src1 (RShiftI src2 src3)));
11176
11177 ins_cost(1.9 * INSN_COST);
11178 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11179
11180 ins_encode %{
11181 __ addw(as_Register($dst$$reg),
11182 as_Register($src1$$reg),
11183 as_Register($src2$$reg),
11184 Assembler::ASR,
11185 $src3$$constant & 0x1f);
11186 %}
11187
11188 ins_pipe(ialu_reg_reg_shift);
11189 %}
11190
11191 // This pattern is automatically generated from aarch64_ad.m4
11192 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11193 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11194 iRegL src1, iRegL src2,
11195 immI src3) %{
11196 match(Set dst (AddL src1 (RShiftL src2 src3)));
11197
11198 ins_cost(1.9 * INSN_COST);
11199 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11200
11201 ins_encode %{
11202 __ add(as_Register($dst$$reg),
11203 as_Register($src1$$reg),
11204 as_Register($src2$$reg),
11205 Assembler::ASR,
11206 $src3$$constant & 0x3f);
11207 %}
11208
11209 ins_pipe(ialu_reg_reg_shift);
11210 %}
11211
11212 // This pattern is automatically generated from aarch64_ad.m4
11213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11214 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11215 iRegIorL2I src1, iRegIorL2I src2,
11216 immI src3) %{
11217 match(Set dst (AddI src1 (LShiftI src2 src3)));
11218
11219 ins_cost(1.9 * INSN_COST);
11220 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11221
11222 ins_encode %{
11223 __ addw(as_Register($dst$$reg),
11224 as_Register($src1$$reg),
11225 as_Register($src2$$reg),
11226 Assembler::LSL,
11227 $src3$$constant & 0x1f);
11228 %}
11229
11230 ins_pipe(ialu_reg_reg_shift);
11231 %}
11232
11233 // This pattern is automatically generated from aarch64_ad.m4
11234 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11235 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11236 iRegL src1, iRegL src2,
11237 immI src3) %{
11238 match(Set dst (AddL src1 (LShiftL src2 src3)));
11239
11240 ins_cost(1.9 * INSN_COST);
11241 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11242
11243 ins_encode %{
11244 __ add(as_Register($dst$$reg),
11245 as_Register($src1$$reg),
11246 as_Register($src2$$reg),
11247 Assembler::LSL,
11248 $src3$$constant & 0x3f);
11249 %}
11250
11251 ins_pipe(ialu_reg_reg_shift);
11252 %}
11253
11254 // This pattern is automatically generated from aarch64_ad.m4
11255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11256 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11257 iRegIorL2I src1, iRegIorL2I src2,
11258 immI src3) %{
11259 match(Set dst (SubI src1 (URShiftI src2 src3)));
11260
11261 ins_cost(1.9 * INSN_COST);
11262 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11263
11264 ins_encode %{
11265 __ subw(as_Register($dst$$reg),
11266 as_Register($src1$$reg),
11267 as_Register($src2$$reg),
11268 Assembler::LSR,
11269 $src3$$constant & 0x1f);
11270 %}
11271
11272 ins_pipe(ialu_reg_reg_shift);
11273 %}
11274
11275 // This pattern is automatically generated from aarch64_ad.m4
11276 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11277 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11278 iRegL src1, iRegL src2,
11279 immI src3) %{
11280 match(Set dst (SubL src1 (URShiftL src2 src3)));
11281
11282 ins_cost(1.9 * INSN_COST);
11283 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11284
11285 ins_encode %{
11286 __ sub(as_Register($dst$$reg),
11287 as_Register($src1$$reg),
11288 as_Register($src2$$reg),
11289 Assembler::LSR,
11290 $src3$$constant & 0x3f);
11291 %}
11292
11293 ins_pipe(ialu_reg_reg_shift);
11294 %}
11295
11296 // This pattern is automatically generated from aarch64_ad.m4
11297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11298 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11299 iRegIorL2I src1, iRegIorL2I src2,
11300 immI src3) %{
11301 match(Set dst (SubI src1 (RShiftI src2 src3)));
11302
11303 ins_cost(1.9 * INSN_COST);
11304 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11305
11306 ins_encode %{
11307 __ subw(as_Register($dst$$reg),
11308 as_Register($src1$$reg),
11309 as_Register($src2$$reg),
11310 Assembler::ASR,
11311 $src3$$constant & 0x1f);
11312 %}
11313
11314 ins_pipe(ialu_reg_reg_shift);
11315 %}
11316
11317 // This pattern is automatically generated from aarch64_ad.m4
11318 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11319 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11320 iRegL src1, iRegL src2,
11321 immI src3) %{
11322 match(Set dst (SubL src1 (RShiftL src2 src3)));
11323
11324 ins_cost(1.9 * INSN_COST);
11325 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11326
11327 ins_encode %{
11328 __ sub(as_Register($dst$$reg),
11329 as_Register($src1$$reg),
11330 as_Register($src2$$reg),
11331 Assembler::ASR,
11332 $src3$$constant & 0x3f);
11333 %}
11334
11335 ins_pipe(ialu_reg_reg_shift);
11336 %}
11337
11338 // This pattern is automatically generated from aarch64_ad.m4
11339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11340 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11341 iRegIorL2I src1, iRegIorL2I src2,
11342 immI src3) %{
11343 match(Set dst (SubI src1 (LShiftI src2 src3)));
11344
11345 ins_cost(1.9 * INSN_COST);
11346 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11347
11348 ins_encode %{
11349 __ subw(as_Register($dst$$reg),
11350 as_Register($src1$$reg),
11351 as_Register($src2$$reg),
11352 Assembler::LSL,
11353 $src3$$constant & 0x1f);
11354 %}
11355
11356 ins_pipe(ialu_reg_reg_shift);
11357 %}
11358
11359 // This pattern is automatically generated from aarch64_ad.m4
11360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11361 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11362 iRegL src1, iRegL src2,
11363 immI src3) %{
11364 match(Set dst (SubL src1 (LShiftL src2 src3)));
11365
11366 ins_cost(1.9 * INSN_COST);
11367 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11368
11369 ins_encode %{
11370 __ sub(as_Register($dst$$reg),
11371 as_Register($src1$$reg),
11372 as_Register($src2$$reg),
11373 Assembler::LSL,
11374 $src3$$constant & 0x3f);
11375 %}
11376
11377 ins_pipe(ialu_reg_reg_shift);
11378 %}
11379
11380 // This pattern is automatically generated from aarch64_ad.m4
11381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11382
11383 // Shift Left followed by Shift Right.
11384 // This idiom is used by the compiler for the i2b bytecode etc.
11385 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11386 %{
11387 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11388 ins_cost(INSN_COST * 2);
11389 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11390 ins_encode %{
11391 int lshift = $lshift_count$$constant & 63;
11392 int rshift = $rshift_count$$constant & 63;
11393 int s = 63 - lshift;
11394 int r = (rshift - lshift) & 63;
11395 __ sbfm(as_Register($dst$$reg),
11396 as_Register($src$$reg),
11397 r, s);
11398 %}
11399
11400 ins_pipe(ialu_reg_shift);
11401 %}
11402
11403 // This pattern is automatically generated from aarch64_ad.m4
11404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11405
11406 // Shift Left followed by Shift Right.
11407 // This idiom is used by the compiler for the i2b bytecode etc.
11408 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11409 %{
11410 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11411 ins_cost(INSN_COST * 2);
11412 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11413 ins_encode %{
11414 int lshift = $lshift_count$$constant & 31;
11415 int rshift = $rshift_count$$constant & 31;
11416 int s = 31 - lshift;
11417 int r = (rshift - lshift) & 31;
11418 __ sbfmw(as_Register($dst$$reg),
11419 as_Register($src$$reg),
11420 r, s);
11421 %}
11422
11423 ins_pipe(ialu_reg_shift);
11424 %}
11425
11426 // This pattern is automatically generated from aarch64_ad.m4
11427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11428
11429 // Shift Left followed by Shift Right.
11430 // This idiom is used by the compiler for the i2b bytecode etc.
11431 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11432 %{
11433 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11434 ins_cost(INSN_COST * 2);
11435 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11436 ins_encode %{
11437 int lshift = $lshift_count$$constant & 63;
11438 int rshift = $rshift_count$$constant & 63;
11439 int s = 63 - lshift;
11440 int r = (rshift - lshift) & 63;
11441 __ ubfm(as_Register($dst$$reg),
11442 as_Register($src$$reg),
11443 r, s);
11444 %}
11445
11446 ins_pipe(ialu_reg_shift);
11447 %}
11448
11449 // This pattern is automatically generated from aarch64_ad.m4
11450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11451
11452 // Shift Left followed by Shift Right.
11453 // This idiom is used by the compiler for the i2b bytecode etc.
11454 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11455 %{
11456 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11457 ins_cost(INSN_COST * 2);
11458 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11459 ins_encode %{
11460 int lshift = $lshift_count$$constant & 31;
11461 int rshift = $rshift_count$$constant & 31;
11462 int s = 31 - lshift;
11463 int r = (rshift - lshift) & 31;
11464 __ ubfmw(as_Register($dst$$reg),
11465 as_Register($src$$reg),
11466 r, s);
11467 %}
11468
11469 ins_pipe(ialu_reg_shift);
11470 %}
11471
11472 // Bitfield extract with shift & mask
11473
11474 // This pattern is automatically generated from aarch64_ad.m4
11475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11476 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11477 %{
11478 match(Set dst (AndI (URShiftI src rshift) mask));
11479 // Make sure we are not going to exceed what ubfxw can do.
11480 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11481
11482 ins_cost(INSN_COST);
11483 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11484 ins_encode %{
11485 int rshift = $rshift$$constant & 31;
11486 intptr_t mask = $mask$$constant;
11487 int width = exact_log2(mask+1);
11488 __ ubfxw(as_Register($dst$$reg),
11489 as_Register($src$$reg), rshift, width);
11490 %}
11491 ins_pipe(ialu_reg_shift);
11492 %}
11493
11494 // This pattern is automatically generated from aarch64_ad.m4
11495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11496 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11497 %{
11498 match(Set dst (AndL (URShiftL src rshift) mask));
11499 // Make sure we are not going to exceed what ubfx can do.
11500 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11501
11502 ins_cost(INSN_COST);
11503 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11504 ins_encode %{
11505 int rshift = $rshift$$constant & 63;
11506 intptr_t mask = $mask$$constant;
11507 int width = exact_log2_long(mask+1);
11508 __ ubfx(as_Register($dst$$reg),
11509 as_Register($src$$reg), rshift, width);
11510 %}
11511 ins_pipe(ialu_reg_shift);
11512 %}
11513
11514
11515 // This pattern is automatically generated from aarch64_ad.m4
11516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11517
11518 // We can use ubfx when extending an And with a mask when we know mask
11519 // is positive. We know that because immI_bitmask guarantees it.
11520 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11521 %{
11522 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11523 // Make sure we are not going to exceed what ubfxw can do.
11524 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11525
11526 ins_cost(INSN_COST * 2);
11527 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11528 ins_encode %{
11529 int rshift = $rshift$$constant & 31;
11530 intptr_t mask = $mask$$constant;
11531 int width = exact_log2(mask+1);
11532 __ ubfx(as_Register($dst$$reg),
11533 as_Register($src$$reg), rshift, width);
11534 %}
11535 ins_pipe(ialu_reg_shift);
11536 %}
11537
11538
11539 // This pattern is automatically generated from aarch64_ad.m4
11540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11541
11542 // We can use ubfiz when masking by a positive number and then left shifting the result.
11543 // We know that the mask is positive because immI_bitmask guarantees it.
11544 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11545 %{
11546 match(Set dst (LShiftI (AndI src mask) lshift));
11547 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11548
11549 ins_cost(INSN_COST);
11550 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11551 ins_encode %{
11552 int lshift = $lshift$$constant & 31;
11553 intptr_t mask = $mask$$constant;
11554 int width = exact_log2(mask+1);
11555 __ ubfizw(as_Register($dst$$reg),
11556 as_Register($src$$reg), lshift, width);
11557 %}
11558 ins_pipe(ialu_reg_shift);
11559 %}
11560
11561 // This pattern is automatically generated from aarch64_ad.m4
11562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11563
11564 // We can use ubfiz when masking by a positive number and then left shifting the result.
11565 // We know that the mask is positive because immL_bitmask guarantees it.
11566 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11567 %{
11568 match(Set dst (LShiftL (AndL src mask) lshift));
11569 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11570
11571 ins_cost(INSN_COST);
11572 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11573 ins_encode %{
11574 int lshift = $lshift$$constant & 63;
11575 intptr_t mask = $mask$$constant;
11576 int width = exact_log2_long(mask+1);
11577 __ ubfiz(as_Register($dst$$reg),
11578 as_Register($src$$reg), lshift, width);
11579 %}
11580 ins_pipe(ialu_reg_shift);
11581 %}
11582
11583 // This pattern is automatically generated from aarch64_ad.m4
11584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11585
11586 // We can use ubfiz when masking by a positive number and then left shifting the result.
11587 // We know that the mask is positive because immI_bitmask guarantees it.
11588 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11589 %{
11590 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11591 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11592
11593 ins_cost(INSN_COST);
11594 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11595 ins_encode %{
11596 int lshift = $lshift$$constant & 31;
11597 intptr_t mask = $mask$$constant;
11598 int width = exact_log2(mask+1);
11599 __ ubfizw(as_Register($dst$$reg),
11600 as_Register($src$$reg), lshift, width);
11601 %}
11602 ins_pipe(ialu_reg_shift);
11603 %}
11604
11605 // This pattern is automatically generated from aarch64_ad.m4
11606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11607
11608 // We can use ubfiz when masking by a positive number and then left shifting the result.
11609 // We know that the mask is positive because immL_bitmask guarantees it.
11610 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11611 %{
11612 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11613 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11614
11615 ins_cost(INSN_COST);
11616 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11617 ins_encode %{
11618 int lshift = $lshift$$constant & 63;
11619 intptr_t mask = $mask$$constant;
11620 int width = exact_log2_long(mask+1);
11621 __ ubfiz(as_Register($dst$$reg),
11622 as_Register($src$$reg), lshift, width);
11623 %}
11624 ins_pipe(ialu_reg_shift);
11625 %}
11626
11627
11628 // This pattern is automatically generated from aarch64_ad.m4
11629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11630
11631 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11632 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11633 %{
11634 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11635 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11636
11637 ins_cost(INSN_COST);
11638 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11639 ins_encode %{
11640 int lshift = $lshift$$constant & 63;
11641 intptr_t mask = $mask$$constant;
11642 int width = exact_log2(mask+1);
11643 __ ubfiz(as_Register($dst$$reg),
11644 as_Register($src$$reg), lshift, width);
11645 %}
11646 ins_pipe(ialu_reg_shift);
11647 %}
11648
11649 // This pattern is automatically generated from aarch64_ad.m4
11650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11651
11652 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11653 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11654 %{
11655 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11656 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11657
11658 ins_cost(INSN_COST);
11659 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11660 ins_encode %{
11661 int lshift = $lshift$$constant & 31;
11662 intptr_t mask = $mask$$constant;
11663 int width = exact_log2(mask+1);
11664 __ ubfiz(as_Register($dst$$reg),
11665 as_Register($src$$reg), lshift, width);
11666 %}
11667 ins_pipe(ialu_reg_shift);
11668 %}
11669
11670 // This pattern is automatically generated from aarch64_ad.m4
11671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11672
11673 // Can skip int2long conversions after AND with small bitmask
11674 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11675 %{
11676 match(Set dst (ConvI2L (AndI src msk)));
11677 ins_cost(INSN_COST);
11678 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11679 ins_encode %{
11680 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11681 %}
11682 ins_pipe(ialu_reg_shift);
11683 %}
11684
11685
11686 // Rotations
11687
11688 // This pattern is automatically generated from aarch64_ad.m4
11689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11690 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11691 %{
11692 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11693 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11694
11695 ins_cost(INSN_COST);
11696 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11697
11698 ins_encode %{
11699 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11700 $rshift$$constant & 63);
11701 %}
11702 ins_pipe(ialu_reg_reg_extr);
11703 %}
11704
11705
11706 // This pattern is automatically generated from aarch64_ad.m4
11707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11708 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11709 %{
11710 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11711 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11712
11713 ins_cost(INSN_COST);
11714 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11715
11716 ins_encode %{
11717 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11718 $rshift$$constant & 31);
11719 %}
11720 ins_pipe(ialu_reg_reg_extr);
11721 %}
11722
11723
11724 // This pattern is automatically generated from aarch64_ad.m4
11725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11726 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11727 %{
11728 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11729 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11730
11731 ins_cost(INSN_COST);
11732 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11733
11734 ins_encode %{
11735 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11736 $rshift$$constant & 63);
11737 %}
11738 ins_pipe(ialu_reg_reg_extr);
11739 %}
11740
11741
11742 // This pattern is automatically generated from aarch64_ad.m4
11743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11744 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11745 %{
11746 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11747 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11748
11749 ins_cost(INSN_COST);
11750 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11751
11752 ins_encode %{
11753 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11754 $rshift$$constant & 31);
11755 %}
11756 ins_pipe(ialu_reg_reg_extr);
11757 %}
11758
11759 // This pattern is automatically generated from aarch64_ad.m4
11760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11761 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11762 %{
11763 match(Set dst (RotateRight src shift));
11764
11765 ins_cost(INSN_COST);
11766 format %{ "ror $dst, $src, $shift" %}
11767
11768 ins_encode %{
11769 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11770 $shift$$constant & 0x1f);
11771 %}
11772 ins_pipe(ialu_reg_reg_vshift);
11773 %}
11774
11775 // This pattern is automatically generated from aarch64_ad.m4
11776 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11777 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11778 %{
11779 match(Set dst (RotateRight src shift));
11780
11781 ins_cost(INSN_COST);
11782 format %{ "ror $dst, $src, $shift" %}
11783
11784 ins_encode %{
11785 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11786 $shift$$constant & 0x3f);
11787 %}
11788 ins_pipe(ialu_reg_reg_vshift);
11789 %}
11790
11791 // This pattern is automatically generated from aarch64_ad.m4
11792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11793 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11794 %{
11795 match(Set dst (RotateRight src shift));
11796
11797 ins_cost(INSN_COST);
11798 format %{ "ror $dst, $src, $shift" %}
11799
11800 ins_encode %{
11801 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11802 %}
11803 ins_pipe(ialu_reg_reg_vshift);
11804 %}
11805
11806 // This pattern is automatically generated from aarch64_ad.m4
11807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11808 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11809 %{
11810 match(Set dst (RotateRight src shift));
11811
11812 ins_cost(INSN_COST);
11813 format %{ "ror $dst, $src, $shift" %}
11814
11815 ins_encode %{
11816 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11817 %}
11818 ins_pipe(ialu_reg_reg_vshift);
11819 %}
11820
11821 // This pattern is automatically generated from aarch64_ad.m4
11822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11823 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11824 %{
11825 match(Set dst (RotateLeft src shift));
11826
11827 ins_cost(INSN_COST);
11828 format %{ "rol $dst, $src, $shift" %}
11829
11830 ins_encode %{
11831 __ subw(rscratch1, zr, as_Register($shift$$reg));
11832 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11833 %}
11834 ins_pipe(ialu_reg_reg_vshift);
11835 %}
11836
11837 // This pattern is automatically generated from aarch64_ad.m4
11838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11839 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11840 %{
11841 match(Set dst (RotateLeft src shift));
11842
11843 ins_cost(INSN_COST);
11844 format %{ "rol $dst, $src, $shift" %}
11845
11846 ins_encode %{
11847 __ subw(rscratch1, zr, as_Register($shift$$reg));
11848 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11849 %}
11850 ins_pipe(ialu_reg_reg_vshift);
11851 %}
11852
11853
11854 // Add/subtract (extended)
11855
11856 // This pattern is automatically generated from aarch64_ad.m4
11857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11858 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11859 %{
11860 match(Set dst (AddL src1 (ConvI2L src2)));
11861 ins_cost(INSN_COST);
11862 format %{ "add $dst, $src1, $src2, sxtw" %}
11863
11864 ins_encode %{
11865 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11866 as_Register($src2$$reg), ext::sxtw);
11867 %}
11868 ins_pipe(ialu_reg_reg);
11869 %}
11870
11871 // This pattern is automatically generated from aarch64_ad.m4
11872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11873 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11874 %{
11875 match(Set dst (SubL src1 (ConvI2L src2)));
11876 ins_cost(INSN_COST);
11877 format %{ "sub $dst, $src1, $src2, sxtw" %}
11878
11879 ins_encode %{
11880 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11881 as_Register($src2$$reg), ext::sxtw);
11882 %}
11883 ins_pipe(ialu_reg_reg);
11884 %}
11885
11886 // This pattern is automatically generated from aarch64_ad.m4
11887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11888 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11889 %{
11890 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11891 ins_cost(INSN_COST);
11892 format %{ "add $dst, $src1, $src2, sxth" %}
11893
11894 ins_encode %{
11895 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11896 as_Register($src2$$reg), ext::sxth);
11897 %}
11898 ins_pipe(ialu_reg_reg);
11899 %}
11900
11901 // This pattern is automatically generated from aarch64_ad.m4
11902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11903 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11904 %{
11905 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11906 ins_cost(INSN_COST);
11907 format %{ "add $dst, $src1, $src2, sxtb" %}
11908
11909 ins_encode %{
11910 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11911 as_Register($src2$$reg), ext::sxtb);
11912 %}
11913 ins_pipe(ialu_reg_reg);
11914 %}
11915
11916 // This pattern is automatically generated from aarch64_ad.m4
11917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11918 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11919 %{
11920 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11921 ins_cost(INSN_COST);
11922 format %{ "add $dst, $src1, $src2, uxtb" %}
11923
11924 ins_encode %{
11925 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11926 as_Register($src2$$reg), ext::uxtb);
11927 %}
11928 ins_pipe(ialu_reg_reg);
11929 %}
11930
11931 // This pattern is automatically generated from aarch64_ad.m4
11932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11933 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11934 %{
11935 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11936 ins_cost(INSN_COST);
11937 format %{ "add $dst, $src1, $src2, sxth" %}
11938
11939 ins_encode %{
11940 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11941 as_Register($src2$$reg), ext::sxth);
11942 %}
11943 ins_pipe(ialu_reg_reg);
11944 %}
11945
11946 // This pattern is automatically generated from aarch64_ad.m4
11947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11948 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11949 %{
11950 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11951 ins_cost(INSN_COST);
11952 format %{ "add $dst, $src1, $src2, sxtw" %}
11953
11954 ins_encode %{
11955 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11956 as_Register($src2$$reg), ext::sxtw);
11957 %}
11958 ins_pipe(ialu_reg_reg);
11959 %}
11960
11961 // This pattern is automatically generated from aarch64_ad.m4
11962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11963 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11964 %{
11965 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11966 ins_cost(INSN_COST);
11967 format %{ "add $dst, $src1, $src2, sxtb" %}
11968
11969 ins_encode %{
11970 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11971 as_Register($src2$$reg), ext::sxtb);
11972 %}
11973 ins_pipe(ialu_reg_reg);
11974 %}
11975
11976 // This pattern is automatically generated from aarch64_ad.m4
11977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11978 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11979 %{
11980 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11981 ins_cost(INSN_COST);
11982 format %{ "add $dst, $src1, $src2, uxtb" %}
11983
11984 ins_encode %{
11985 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11986 as_Register($src2$$reg), ext::uxtb);
11987 %}
11988 ins_pipe(ialu_reg_reg);
11989 %}
11990
11991 // This pattern is automatically generated from aarch64_ad.m4
11992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11993 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11994 %{
11995 match(Set dst (AddI src1 (AndI src2 mask)));
11996 ins_cost(INSN_COST);
11997 format %{ "addw $dst, $src1, $src2, uxtb" %}
11998
11999 ins_encode %{
12000 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12001 as_Register($src2$$reg), ext::uxtb);
12002 %}
12003 ins_pipe(ialu_reg_reg);
12004 %}
12005
12006 // This pattern is automatically generated from aarch64_ad.m4
12007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12008 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12009 %{
12010 match(Set dst (AddI src1 (AndI src2 mask)));
12011 ins_cost(INSN_COST);
12012 format %{ "addw $dst, $src1, $src2, uxth" %}
12013
12014 ins_encode %{
12015 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12016 as_Register($src2$$reg), ext::uxth);
12017 %}
12018 ins_pipe(ialu_reg_reg);
12019 %}
12020
12021 // This pattern is automatically generated from aarch64_ad.m4
12022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12023 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12024 %{
12025 match(Set dst (AddL src1 (AndL src2 mask)));
12026 ins_cost(INSN_COST);
12027 format %{ "add $dst, $src1, $src2, uxtb" %}
12028
12029 ins_encode %{
12030 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12031 as_Register($src2$$reg), ext::uxtb);
12032 %}
12033 ins_pipe(ialu_reg_reg);
12034 %}
12035
12036 // This pattern is automatically generated from aarch64_ad.m4
12037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12038 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12039 %{
12040 match(Set dst (AddL src1 (AndL src2 mask)));
12041 ins_cost(INSN_COST);
12042 format %{ "add $dst, $src1, $src2, uxth" %}
12043
12044 ins_encode %{
12045 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12046 as_Register($src2$$reg), ext::uxth);
12047 %}
12048 ins_pipe(ialu_reg_reg);
12049 %}
12050
12051 // This pattern is automatically generated from aarch64_ad.m4
12052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12053 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12054 %{
12055 match(Set dst (AddL src1 (AndL src2 mask)));
12056 ins_cost(INSN_COST);
12057 format %{ "add $dst, $src1, $src2, uxtw" %}
12058
12059 ins_encode %{
12060 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12061 as_Register($src2$$reg), ext::uxtw);
12062 %}
12063 ins_pipe(ialu_reg_reg);
12064 %}
12065
12066 // This pattern is automatically generated from aarch64_ad.m4
12067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12068 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12069 %{
12070 match(Set dst (SubI src1 (AndI src2 mask)));
12071 ins_cost(INSN_COST);
12072 format %{ "subw $dst, $src1, $src2, uxtb" %}
12073
12074 ins_encode %{
12075 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12076 as_Register($src2$$reg), ext::uxtb);
12077 %}
12078 ins_pipe(ialu_reg_reg);
12079 %}
12080
12081 // This pattern is automatically generated from aarch64_ad.m4
12082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12083 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12084 %{
12085 match(Set dst (SubI src1 (AndI src2 mask)));
12086 ins_cost(INSN_COST);
12087 format %{ "subw $dst, $src1, $src2, uxth" %}
12088
12089 ins_encode %{
12090 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12091 as_Register($src2$$reg), ext::uxth);
12092 %}
12093 ins_pipe(ialu_reg_reg);
12094 %}
12095
12096 // This pattern is automatically generated from aarch64_ad.m4
12097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12098 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12099 %{
12100 match(Set dst (SubL src1 (AndL src2 mask)));
12101 ins_cost(INSN_COST);
12102 format %{ "sub $dst, $src1, $src2, uxtb" %}
12103
12104 ins_encode %{
12105 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12106 as_Register($src2$$reg), ext::uxtb);
12107 %}
12108 ins_pipe(ialu_reg_reg);
12109 %}
12110
12111 // This pattern is automatically generated from aarch64_ad.m4
12112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12113 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12114 %{
12115 match(Set dst (SubL src1 (AndL src2 mask)));
12116 ins_cost(INSN_COST);
12117 format %{ "sub $dst, $src1, $src2, uxth" %}
12118
12119 ins_encode %{
12120 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12121 as_Register($src2$$reg), ext::uxth);
12122 %}
12123 ins_pipe(ialu_reg_reg);
12124 %}
12125
12126 // This pattern is automatically generated from aarch64_ad.m4
12127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12128 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12129 %{
12130 match(Set dst (SubL src1 (AndL src2 mask)));
12131 ins_cost(INSN_COST);
12132 format %{ "sub $dst, $src1, $src2, uxtw" %}
12133
12134 ins_encode %{
12135 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12136 as_Register($src2$$reg), ext::uxtw);
12137 %}
12138 ins_pipe(ialu_reg_reg);
12139 %}
12140
12141
12142 // This pattern is automatically generated from aarch64_ad.m4
12143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12144 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12145 %{
12146 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12147 ins_cost(1.9 * INSN_COST);
12148 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12149
12150 ins_encode %{
12151 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12152 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12153 %}
12154 ins_pipe(ialu_reg_reg_shift);
12155 %}
12156
12157 // This pattern is automatically generated from aarch64_ad.m4
12158 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12159 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12160 %{
12161 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12162 ins_cost(1.9 * INSN_COST);
12163 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12164
12165 ins_encode %{
12166 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12167 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12168 %}
12169 ins_pipe(ialu_reg_reg_shift);
12170 %}
12171
12172 // This pattern is automatically generated from aarch64_ad.m4
12173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12174 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12175 %{
12176 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12177 ins_cost(1.9 * INSN_COST);
12178 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12179
12180 ins_encode %{
12181 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12182 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12183 %}
12184 ins_pipe(ialu_reg_reg_shift);
12185 %}
12186
12187 // This pattern is automatically generated from aarch64_ad.m4
12188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12189 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12190 %{
12191 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12192 ins_cost(1.9 * INSN_COST);
12193 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12194
12195 ins_encode %{
12196 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12197 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12198 %}
12199 ins_pipe(ialu_reg_reg_shift);
12200 %}
12201
12202 // This pattern is automatically generated from aarch64_ad.m4
12203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12204 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12205 %{
12206 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12207 ins_cost(1.9 * INSN_COST);
12208 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12209
12210 ins_encode %{
12211 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12212 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12213 %}
12214 ins_pipe(ialu_reg_reg_shift);
12215 %}
12216
12217 // This pattern is automatically generated from aarch64_ad.m4
12218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12219 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12220 %{
12221 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12222 ins_cost(1.9 * INSN_COST);
12223 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12224
12225 ins_encode %{
12226 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12227 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12228 %}
12229 ins_pipe(ialu_reg_reg_shift);
12230 %}
12231
12232 // This pattern is automatically generated from aarch64_ad.m4
12233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12234 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12235 %{
12236 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12237 ins_cost(1.9 * INSN_COST);
12238 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12239
12240 ins_encode %{
12241 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12242 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12243 %}
12244 ins_pipe(ialu_reg_reg_shift);
12245 %}
12246
12247 // This pattern is automatically generated from aarch64_ad.m4
12248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12249 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12250 %{
12251 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12252 ins_cost(1.9 * INSN_COST);
12253 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12254
12255 ins_encode %{
12256 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12257 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12258 %}
12259 ins_pipe(ialu_reg_reg_shift);
12260 %}
12261
12262 // This pattern is automatically generated from aarch64_ad.m4
12263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12264 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12265 %{
12266 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12267 ins_cost(1.9 * INSN_COST);
12268 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12269
12270 ins_encode %{
12271 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12272 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12273 %}
12274 ins_pipe(ialu_reg_reg_shift);
12275 %}
12276
12277 // This pattern is automatically generated from aarch64_ad.m4
12278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12279 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12280 %{
12281 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12282 ins_cost(1.9 * INSN_COST);
12283 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12284
12285 ins_encode %{
12286 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12287 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12288 %}
12289 ins_pipe(ialu_reg_reg_shift);
12290 %}
12291
12292 // This pattern is automatically generated from aarch64_ad.m4
12293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12294 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12295 %{
12296 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12297 ins_cost(1.9 * INSN_COST);
12298 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12299
12300 ins_encode %{
12301 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12302 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12303 %}
12304 ins_pipe(ialu_reg_reg_shift);
12305 %}
12306
12307 // This pattern is automatically generated from aarch64_ad.m4
12308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12309 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12310 %{
12311 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12312 ins_cost(1.9 * INSN_COST);
12313 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12314
12315 ins_encode %{
12316 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12317 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12318 %}
12319 ins_pipe(ialu_reg_reg_shift);
12320 %}
12321
12322 // This pattern is automatically generated from aarch64_ad.m4
12323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12324 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12325 %{
12326 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12327 ins_cost(1.9 * INSN_COST);
12328 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12329
12330 ins_encode %{
12331 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12332 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12333 %}
12334 ins_pipe(ialu_reg_reg_shift);
12335 %}
12336
12337 // This pattern is automatically generated from aarch64_ad.m4
12338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12339 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12340 %{
12341 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12342 ins_cost(1.9 * INSN_COST);
12343 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12344
12345 ins_encode %{
12346 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12347 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12348 %}
12349 ins_pipe(ialu_reg_reg_shift);
12350 %}
12351
12352 // This pattern is automatically generated from aarch64_ad.m4
12353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12354 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12355 %{
12356 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12357 ins_cost(1.9 * INSN_COST);
12358 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12359
12360 ins_encode %{
12361 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12362 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12363 %}
12364 ins_pipe(ialu_reg_reg_shift);
12365 %}
12366
12367 // This pattern is automatically generated from aarch64_ad.m4
12368 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12369 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12370 %{
12371 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12372 ins_cost(1.9 * INSN_COST);
12373 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12374
12375 ins_encode %{
12376 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12377 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12378 %}
12379 ins_pipe(ialu_reg_reg_shift);
12380 %}
12381
12382 // This pattern is automatically generated from aarch64_ad.m4
12383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12384 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12385 %{
12386 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12387 ins_cost(1.9 * INSN_COST);
12388 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12389
12390 ins_encode %{
12391 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12392 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12393 %}
12394 ins_pipe(ialu_reg_reg_shift);
12395 %}
12396
12397 // This pattern is automatically generated from aarch64_ad.m4
12398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12399 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12400 %{
12401 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12402 ins_cost(1.9 * INSN_COST);
12403 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12404
12405 ins_encode %{
12406 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12407 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12408 %}
12409 ins_pipe(ialu_reg_reg_shift);
12410 %}
12411
12412 // This pattern is automatically generated from aarch64_ad.m4
12413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12414 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12415 %{
12416 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12417 ins_cost(1.9 * INSN_COST);
12418 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12419
12420 ins_encode %{
12421 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12422 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12423 %}
12424 ins_pipe(ialu_reg_reg_shift);
12425 %}
12426
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12430 %{
12431 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12432 ins_cost(1.9 * INSN_COST);
12433 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12434
12435 ins_encode %{
12436 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12437 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12438 %}
12439 ins_pipe(ialu_reg_reg_shift);
12440 %}
12441
12442 // This pattern is automatically generated from aarch64_ad.m4
12443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12444 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12445 %{
12446 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12447 ins_cost(1.9 * INSN_COST);
12448 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12449
12450 ins_encode %{
12451 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12452 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12453 %}
12454 ins_pipe(ialu_reg_reg_shift);
12455 %}
12456
12457 // This pattern is automatically generated from aarch64_ad.m4
12458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12459 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12460 %{
12461 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12462 ins_cost(1.9 * INSN_COST);
12463 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12464
12465 ins_encode %{
12466 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12467 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12468 %}
12469 ins_pipe(ialu_reg_reg_shift);
12470 %}
12471
12472 // This pattern is automatically generated from aarch64_ad.m4
12473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12474 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12475 %{
12476 effect(DEF dst, USE src1, USE src2, USE cr);
12477 ins_cost(INSN_COST * 2);
12478 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12479
12480 ins_encode %{
12481 __ cselw($dst$$Register,
12482 $src1$$Register,
12483 $src2$$Register,
12484 Assembler::LT);
12485 %}
12486 ins_pipe(icond_reg_reg);
12487 %}
12488
12489 // This pattern is automatically generated from aarch64_ad.m4
12490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12491 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12492 %{
12493 effect(DEF dst, USE src1, USE src2, USE cr);
12494 ins_cost(INSN_COST * 2);
12495 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12496
12497 ins_encode %{
12498 __ cselw($dst$$Register,
12499 $src1$$Register,
12500 $src2$$Register,
12501 Assembler::GT);
12502 %}
12503 ins_pipe(icond_reg_reg);
12504 %}
12505
12506 // This pattern is automatically generated from aarch64_ad.m4
12507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12508 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12509 %{
12510 effect(DEF dst, USE src1, USE cr);
12511 ins_cost(INSN_COST * 2);
12512 format %{ "cselw $dst, $src1, zr lt\t" %}
12513
12514 ins_encode %{
12515 __ cselw($dst$$Register,
12516 $src1$$Register,
12517 zr,
12518 Assembler::LT);
12519 %}
12520 ins_pipe(icond_reg);
12521 %}
12522
12523 // This pattern is automatically generated from aarch64_ad.m4
12524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12525 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12526 %{
12527 effect(DEF dst, USE src1, USE cr);
12528 ins_cost(INSN_COST * 2);
12529 format %{ "cselw $dst, $src1, zr gt\t" %}
12530
12531 ins_encode %{
12532 __ cselw($dst$$Register,
12533 $src1$$Register,
12534 zr,
12535 Assembler::GT);
12536 %}
12537 ins_pipe(icond_reg);
12538 %}
12539
12540 // This pattern is automatically generated from aarch64_ad.m4
12541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12542 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12543 %{
12544 effect(DEF dst, USE src1, USE cr);
12545 ins_cost(INSN_COST * 2);
12546 format %{ "csincw $dst, $src1, zr le\t" %}
12547
12548 ins_encode %{
12549 __ csincw($dst$$Register,
12550 $src1$$Register,
12551 zr,
12552 Assembler::LE);
12553 %}
12554 ins_pipe(icond_reg);
12555 %}
12556
12557 // This pattern is automatically generated from aarch64_ad.m4
12558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12559 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12560 %{
12561 effect(DEF dst, USE src1, USE cr);
12562 ins_cost(INSN_COST * 2);
12563 format %{ "csincw $dst, $src1, zr gt\t" %}
12564
12565 ins_encode %{
12566 __ csincw($dst$$Register,
12567 $src1$$Register,
12568 zr,
12569 Assembler::GT);
12570 %}
12571 ins_pipe(icond_reg);
12572 %}
12573
12574 // This pattern is automatically generated from aarch64_ad.m4
12575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12576 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12577 %{
12578 effect(DEF dst, USE src1, USE cr);
12579 ins_cost(INSN_COST * 2);
12580 format %{ "csinvw $dst, $src1, zr lt\t" %}
12581
12582 ins_encode %{
12583 __ csinvw($dst$$Register,
12584 $src1$$Register,
12585 zr,
12586 Assembler::LT);
12587 %}
12588 ins_pipe(icond_reg);
12589 %}
12590
12591 // This pattern is automatically generated from aarch64_ad.m4
12592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12593 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12594 %{
12595 effect(DEF dst, USE src1, USE cr);
12596 ins_cost(INSN_COST * 2);
12597 format %{ "csinvw $dst, $src1, zr ge\t" %}
12598
12599 ins_encode %{
12600 __ csinvw($dst$$Register,
12601 $src1$$Register,
12602 zr,
12603 Assembler::GE);
12604 %}
12605 ins_pipe(icond_reg);
12606 %}
12607
12608 // This pattern is automatically generated from aarch64_ad.m4
12609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12610 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12611 %{
12612 match(Set dst (MinI src imm));
12613 ins_cost(INSN_COST * 3);
12614 expand %{
12615 rFlagsReg cr;
12616 compI_reg_imm0(cr, src);
12617 cmovI_reg_imm0_lt(dst, src, cr);
12618 %}
12619 %}
12620
12621 // This pattern is automatically generated from aarch64_ad.m4
12622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12623 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12624 %{
12625 match(Set dst (MinI imm src));
12626 ins_cost(INSN_COST * 3);
12627 expand %{
12628 rFlagsReg cr;
12629 compI_reg_imm0(cr, src);
12630 cmovI_reg_imm0_lt(dst, src, cr);
12631 %}
12632 %}
12633
12634 // This pattern is automatically generated from aarch64_ad.m4
12635 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12636 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12637 %{
12638 match(Set dst (MinI src imm));
12639 ins_cost(INSN_COST * 3);
12640 expand %{
12641 rFlagsReg cr;
12642 compI_reg_imm0(cr, src);
12643 cmovI_reg_imm1_le(dst, src, cr);
12644 %}
12645 %}
12646
12647 // This pattern is automatically generated from aarch64_ad.m4
12648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12649 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12650 %{
12651 match(Set dst (MinI imm src));
12652 ins_cost(INSN_COST * 3);
12653 expand %{
12654 rFlagsReg cr;
12655 compI_reg_imm0(cr, src);
12656 cmovI_reg_imm1_le(dst, src, cr);
12657 %}
12658 %}
12659
12660 // This pattern is automatically generated from aarch64_ad.m4
12661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12662 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12663 %{
12664 match(Set dst (MinI src imm));
12665 ins_cost(INSN_COST * 3);
12666 expand %{
12667 rFlagsReg cr;
12668 compI_reg_imm0(cr, src);
12669 cmovI_reg_immM1_lt(dst, src, cr);
12670 %}
12671 %}
12672
12673 // This pattern is automatically generated from aarch64_ad.m4
12674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12675 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12676 %{
12677 match(Set dst (MinI imm src));
12678 ins_cost(INSN_COST * 3);
12679 expand %{
12680 rFlagsReg cr;
12681 compI_reg_imm0(cr, src);
12682 cmovI_reg_immM1_lt(dst, src, cr);
12683 %}
12684 %}
12685
12686 // This pattern is automatically generated from aarch64_ad.m4
12687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12688 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12689 %{
12690 match(Set dst (MaxI src imm));
12691 ins_cost(INSN_COST * 3);
12692 expand %{
12693 rFlagsReg cr;
12694 compI_reg_imm0(cr, src);
12695 cmovI_reg_imm0_gt(dst, src, cr);
12696 %}
12697 %}
12698
12699 // This pattern is automatically generated from aarch64_ad.m4
12700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12701 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12702 %{
12703 match(Set dst (MaxI imm src));
12704 ins_cost(INSN_COST * 3);
12705 expand %{
12706 rFlagsReg cr;
12707 compI_reg_imm0(cr, src);
12708 cmovI_reg_imm0_gt(dst, src, cr);
12709 %}
12710 %}
12711
12712 // This pattern is automatically generated from aarch64_ad.m4
12713 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12714 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12715 %{
12716 match(Set dst (MaxI src imm));
12717 ins_cost(INSN_COST * 3);
12718 expand %{
12719 rFlagsReg cr;
12720 compI_reg_imm0(cr, src);
12721 cmovI_reg_imm1_gt(dst, src, cr);
12722 %}
12723 %}
12724
12725 // This pattern is automatically generated from aarch64_ad.m4
12726 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12727 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12728 %{
12729 match(Set dst (MaxI imm src));
12730 ins_cost(INSN_COST * 3);
12731 expand %{
12732 rFlagsReg cr;
12733 compI_reg_imm0(cr, src);
12734 cmovI_reg_imm1_gt(dst, src, cr);
12735 %}
12736 %}
12737
12738 // This pattern is automatically generated from aarch64_ad.m4
12739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12740 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12741 %{
12742 match(Set dst (MaxI src imm));
12743 ins_cost(INSN_COST * 3);
12744 expand %{
12745 rFlagsReg cr;
12746 compI_reg_imm0(cr, src);
12747 cmovI_reg_immM1_ge(dst, src, cr);
12748 %}
12749 %}
12750
12751 // This pattern is automatically generated from aarch64_ad.m4
12752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12753 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12754 %{
12755 match(Set dst (MaxI imm src));
12756 ins_cost(INSN_COST * 3);
12757 expand %{
12758 rFlagsReg cr;
12759 compI_reg_imm0(cr, src);
12760 cmovI_reg_immM1_ge(dst, src, cr);
12761 %}
12762 %}
12763
12764 // This pattern is automatically generated from aarch64_ad.m4
12765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12766 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12767 %{
12768 match(Set dst (ReverseI src));
12769 ins_cost(INSN_COST);
12770 format %{ "rbitw $dst, $src" %}
12771 ins_encode %{
12772 __ rbitw($dst$$Register, $src$$Register);
12773 %}
12774 ins_pipe(ialu_reg);
12775 %}
12776
12777 // This pattern is automatically generated from aarch64_ad.m4
12778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12779 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12780 %{
12781 match(Set dst (ReverseL src));
12782 ins_cost(INSN_COST);
12783 format %{ "rbit $dst, $src" %}
12784 ins_encode %{
12785 __ rbit($dst$$Register, $src$$Register);
12786 %}
12787 ins_pipe(ialu_reg);
12788 %}
12789
12790
12791 // END This section of the file is automatically generated. Do not edit --------------
12792
12793
12794 // ============================================================================
12795 // Floating Point Arithmetic Instructions
12796
12797 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12798 match(Set dst (AddHF src1 src2));
12799 format %{ "faddh $dst, $src1, $src2" %}
12800 ins_encode %{
12801 __ faddh($dst$$FloatRegister,
12802 $src1$$FloatRegister,
12803 $src2$$FloatRegister);
12804 %}
12805 ins_pipe(fp_dop_reg_reg_s);
12806 %}
12807
12808 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12809 match(Set dst (AddF src1 src2));
12810
12811 ins_cost(INSN_COST * 5);
12812 format %{ "fadds $dst, $src1, $src2" %}
12813
12814 ins_encode %{
12815 __ fadds(as_FloatRegister($dst$$reg),
12816 as_FloatRegister($src1$$reg),
12817 as_FloatRegister($src2$$reg));
12818 %}
12819
12820 ins_pipe(fp_dop_reg_reg_s);
12821 %}
12822
12823 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12824 match(Set dst (AddD src1 src2));
12825
12826 ins_cost(INSN_COST * 5);
12827 format %{ "faddd $dst, $src1, $src2" %}
12828
12829 ins_encode %{
12830 __ faddd(as_FloatRegister($dst$$reg),
12831 as_FloatRegister($src1$$reg),
12832 as_FloatRegister($src2$$reg));
12833 %}
12834
12835 ins_pipe(fp_dop_reg_reg_d);
12836 %}
12837
12838 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12839 match(Set dst (SubHF src1 src2));
12840 format %{ "fsubh $dst, $src1, $src2" %}
12841 ins_encode %{
12842 __ fsubh($dst$$FloatRegister,
12843 $src1$$FloatRegister,
12844 $src2$$FloatRegister);
12845 %}
12846 ins_pipe(fp_dop_reg_reg_s);
12847 %}
12848
12849 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12850 match(Set dst (SubF src1 src2));
12851
12852 ins_cost(INSN_COST * 5);
12853 format %{ "fsubs $dst, $src1, $src2" %}
12854
12855 ins_encode %{
12856 __ fsubs(as_FloatRegister($dst$$reg),
12857 as_FloatRegister($src1$$reg),
12858 as_FloatRegister($src2$$reg));
12859 %}
12860
12861 ins_pipe(fp_dop_reg_reg_s);
12862 %}
12863
12864 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12865 match(Set dst (SubD src1 src2));
12866
12867 ins_cost(INSN_COST * 5);
12868 format %{ "fsubd $dst, $src1, $src2" %}
12869
12870 ins_encode %{
12871 __ fsubd(as_FloatRegister($dst$$reg),
12872 as_FloatRegister($src1$$reg),
12873 as_FloatRegister($src2$$reg));
12874 %}
12875
12876 ins_pipe(fp_dop_reg_reg_d);
12877 %}
12878
12879 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12880 match(Set dst (MulHF src1 src2));
12881 format %{ "fmulh $dst, $src1, $src2" %}
12882 ins_encode %{
12883 __ fmulh($dst$$FloatRegister,
12884 $src1$$FloatRegister,
12885 $src2$$FloatRegister);
12886 %}
12887 ins_pipe(fp_dop_reg_reg_s);
12888 %}
12889
12890 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12891 match(Set dst (MulF src1 src2));
12892
12893 ins_cost(INSN_COST * 6);
12894 format %{ "fmuls $dst, $src1, $src2" %}
12895
12896 ins_encode %{
12897 __ fmuls(as_FloatRegister($dst$$reg),
12898 as_FloatRegister($src1$$reg),
12899 as_FloatRegister($src2$$reg));
12900 %}
12901
12902 ins_pipe(fp_dop_reg_reg_s);
12903 %}
12904
12905 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12906 match(Set dst (MulD src1 src2));
12907
12908 ins_cost(INSN_COST * 6);
12909 format %{ "fmuld $dst, $src1, $src2" %}
12910
12911 ins_encode %{
12912 __ fmuld(as_FloatRegister($dst$$reg),
12913 as_FloatRegister($src1$$reg),
12914 as_FloatRegister($src2$$reg));
12915 %}
12916
12917 ins_pipe(fp_dop_reg_reg_d);
12918 %}
12919
12920 // src1 * src2 + src3 (half-precision float)
12921 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12922 match(Set dst (FmaHF src3 (Binary src1 src2)));
12923 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12924 ins_encode %{
12925 assert(UseFMA, "Needs FMA instructions support.");
12926 __ fmaddh($dst$$FloatRegister,
12927 $src1$$FloatRegister,
12928 $src2$$FloatRegister,
12929 $src3$$FloatRegister);
12930 %}
12931 ins_pipe(pipe_class_default);
12932 %}
12933
12934 // src1 * src2 + src3
12935 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12936 match(Set dst (FmaF src3 (Binary src1 src2)));
12937
12938 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12939
12940 ins_encode %{
12941 assert(UseFMA, "Needs FMA instructions support.");
12942 __ fmadds(as_FloatRegister($dst$$reg),
12943 as_FloatRegister($src1$$reg),
12944 as_FloatRegister($src2$$reg),
12945 as_FloatRegister($src3$$reg));
12946 %}
12947
12948 ins_pipe(pipe_class_default);
12949 %}
12950
12951 // src1 * src2 + src3
12952 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12953 match(Set dst (FmaD src3 (Binary src1 src2)));
12954
12955 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12956
12957 ins_encode %{
12958 assert(UseFMA, "Needs FMA instructions support.");
12959 __ fmaddd(as_FloatRegister($dst$$reg),
12960 as_FloatRegister($src1$$reg),
12961 as_FloatRegister($src2$$reg),
12962 as_FloatRegister($src3$$reg));
12963 %}
12964
12965 ins_pipe(pipe_class_default);
12966 %}
12967
12968 // src1 * (-src2) + src3
12969 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12970 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12971 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12972
12973 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12974
12975 ins_encode %{
12976 assert(UseFMA, "Needs FMA instructions support.");
12977 __ fmsubs(as_FloatRegister($dst$$reg),
12978 as_FloatRegister($src1$$reg),
12979 as_FloatRegister($src2$$reg),
12980 as_FloatRegister($src3$$reg));
12981 %}
12982
12983 ins_pipe(pipe_class_default);
12984 %}
12985
12986 // src1 * (-src2) + src3
12987 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12988 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12989 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12990
12991 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12992
12993 ins_encode %{
12994 assert(UseFMA, "Needs FMA instructions support.");
12995 __ fmsubd(as_FloatRegister($dst$$reg),
12996 as_FloatRegister($src1$$reg),
12997 as_FloatRegister($src2$$reg),
12998 as_FloatRegister($src3$$reg));
12999 %}
13000
13001 ins_pipe(pipe_class_default);
13002 %}
13003
13004 // src1 * (-src2) - src3
13005 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13006 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13007 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13008
13009 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13010
13011 ins_encode %{
13012 assert(UseFMA, "Needs FMA instructions support.");
13013 __ fnmadds(as_FloatRegister($dst$$reg),
13014 as_FloatRegister($src1$$reg),
13015 as_FloatRegister($src2$$reg),
13016 as_FloatRegister($src3$$reg));
13017 %}
13018
13019 ins_pipe(pipe_class_default);
13020 %}
13021
13022 // src1 * (-src2) - src3
13023 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13024 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13025 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13026
13027 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13028
13029 ins_encode %{
13030 assert(UseFMA, "Needs FMA instructions support.");
13031 __ fnmaddd(as_FloatRegister($dst$$reg),
13032 as_FloatRegister($src1$$reg),
13033 as_FloatRegister($src2$$reg),
13034 as_FloatRegister($src3$$reg));
13035 %}
13036
13037 ins_pipe(pipe_class_default);
13038 %}
13039
13040 // src1 * src2 - src3
13041 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13042 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13043
13044 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13045
13046 ins_encode %{
13047 assert(UseFMA, "Needs FMA instructions support.");
13048 __ fnmsubs(as_FloatRegister($dst$$reg),
13049 as_FloatRegister($src1$$reg),
13050 as_FloatRegister($src2$$reg),
13051 as_FloatRegister($src3$$reg));
13052 %}
13053
13054 ins_pipe(pipe_class_default);
13055 %}
13056
13057 // src1 * src2 - src3
13058 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13059 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13060
13061 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13062
13063 ins_encode %{
13064 assert(UseFMA, "Needs FMA instructions support.");
13065 // n.b. insn name should be fnmsubd
13066 __ fnmsub(as_FloatRegister($dst$$reg),
13067 as_FloatRegister($src1$$reg),
13068 as_FloatRegister($src2$$reg),
13069 as_FloatRegister($src3$$reg));
13070 %}
13071
13072 ins_pipe(pipe_class_default);
13073 %}
13074
13075 // Math.max(HH)H (half-precision float)
13076 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13077 match(Set dst (MaxHF src1 src2));
13078 format %{ "fmaxh $dst, $src1, $src2" %}
13079 ins_encode %{
13080 __ fmaxh($dst$$FloatRegister,
13081 $src1$$FloatRegister,
13082 $src2$$FloatRegister);
13083 %}
13084 ins_pipe(fp_dop_reg_reg_s);
13085 %}
13086
13087 // Math.min(HH)H (half-precision float)
13088 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13089 match(Set dst (MinHF src1 src2));
13090 format %{ "fminh $dst, $src1, $src2" %}
13091 ins_encode %{
13092 __ fminh($dst$$FloatRegister,
13093 $src1$$FloatRegister,
13094 $src2$$FloatRegister);
13095 %}
13096 ins_pipe(fp_dop_reg_reg_s);
13097 %}
13098
13099 // Math.max(FF)F
13100 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13101 match(Set dst (MaxF src1 src2));
13102
13103 format %{ "fmaxs $dst, $src1, $src2" %}
13104 ins_encode %{
13105 __ fmaxs(as_FloatRegister($dst$$reg),
13106 as_FloatRegister($src1$$reg),
13107 as_FloatRegister($src2$$reg));
13108 %}
13109
13110 ins_pipe(fp_dop_reg_reg_s);
13111 %}
13112
13113 // Math.min(FF)F
13114 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13115 match(Set dst (MinF src1 src2));
13116
13117 format %{ "fmins $dst, $src1, $src2" %}
13118 ins_encode %{
13119 __ fmins(as_FloatRegister($dst$$reg),
13120 as_FloatRegister($src1$$reg),
13121 as_FloatRegister($src2$$reg));
13122 %}
13123
13124 ins_pipe(fp_dop_reg_reg_s);
13125 %}
13126
13127 // Math.max(DD)D
13128 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13129 match(Set dst (MaxD src1 src2));
13130
13131 format %{ "fmaxd $dst, $src1, $src2" %}
13132 ins_encode %{
13133 __ fmaxd(as_FloatRegister($dst$$reg),
13134 as_FloatRegister($src1$$reg),
13135 as_FloatRegister($src2$$reg));
13136 %}
13137
13138 ins_pipe(fp_dop_reg_reg_d);
13139 %}
13140
13141 // Math.min(DD)D
13142 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13143 match(Set dst (MinD src1 src2));
13144
13145 format %{ "fmind $dst, $src1, $src2" %}
13146 ins_encode %{
13147 __ fmind(as_FloatRegister($dst$$reg),
13148 as_FloatRegister($src1$$reg),
13149 as_FloatRegister($src2$$reg));
13150 %}
13151
13152 ins_pipe(fp_dop_reg_reg_d);
13153 %}
13154
13155 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13156 match(Set dst (DivHF src1 src2));
13157 format %{ "fdivh $dst, $src1, $src2" %}
13158 ins_encode %{
13159 __ fdivh($dst$$FloatRegister,
13160 $src1$$FloatRegister,
13161 $src2$$FloatRegister);
13162 %}
13163 ins_pipe(fp_div_s);
13164 %}
13165
13166 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13167 match(Set dst (DivF src1 src2));
13168
13169 ins_cost(INSN_COST * 18);
13170 format %{ "fdivs $dst, $src1, $src2" %}
13171
13172 ins_encode %{
13173 __ fdivs(as_FloatRegister($dst$$reg),
13174 as_FloatRegister($src1$$reg),
13175 as_FloatRegister($src2$$reg));
13176 %}
13177
13178 ins_pipe(fp_div_s);
13179 %}
13180
13181 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13182 match(Set dst (DivD src1 src2));
13183
13184 ins_cost(INSN_COST * 32);
13185 format %{ "fdivd $dst, $src1, $src2" %}
13186
13187 ins_encode %{
13188 __ fdivd(as_FloatRegister($dst$$reg),
13189 as_FloatRegister($src1$$reg),
13190 as_FloatRegister($src2$$reg));
13191 %}
13192
13193 ins_pipe(fp_div_d);
13194 %}
13195
13196 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13197 match(Set dst (NegF src));
13198
13199 ins_cost(INSN_COST * 3);
13200 format %{ "fneg $dst, $src" %}
13201
13202 ins_encode %{
13203 __ fnegs(as_FloatRegister($dst$$reg),
13204 as_FloatRegister($src$$reg));
13205 %}
13206
13207 ins_pipe(fp_uop_s);
13208 %}
13209
13210 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13211 match(Set dst (NegD src));
13212
13213 ins_cost(INSN_COST * 3);
13214 format %{ "fnegd $dst, $src" %}
13215
13216 ins_encode %{
13217 __ fnegd(as_FloatRegister($dst$$reg),
13218 as_FloatRegister($src$$reg));
13219 %}
13220
13221 ins_pipe(fp_uop_d);
13222 %}
13223
13224 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13225 %{
13226 match(Set dst (AbsI src));
13227
13228 effect(KILL cr);
13229 ins_cost(INSN_COST * 2);
13230 format %{ "cmpw $src, zr\n\t"
13231 "cnegw $dst, $src, Assembler::LT\t# int abs"
13232 %}
13233
13234 ins_encode %{
13235 __ cmpw(as_Register($src$$reg), zr);
13236 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13237 %}
13238 ins_pipe(pipe_class_default);
13239 %}
13240
13241 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13242 %{
13243 match(Set dst (AbsL src));
13244
13245 effect(KILL cr);
13246 ins_cost(INSN_COST * 2);
13247 format %{ "cmp $src, zr\n\t"
13248 "cneg $dst, $src, Assembler::LT\t# long abs"
13249 %}
13250
13251 ins_encode %{
13252 __ cmp(as_Register($src$$reg), zr);
13253 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13254 %}
13255 ins_pipe(pipe_class_default);
13256 %}
13257
13258 instruct absF_reg(vRegF dst, vRegF src) %{
13259 match(Set dst (AbsF src));
13260
13261 ins_cost(INSN_COST * 3);
13262 format %{ "fabss $dst, $src" %}
13263 ins_encode %{
13264 __ fabss(as_FloatRegister($dst$$reg),
13265 as_FloatRegister($src$$reg));
13266 %}
13267
13268 ins_pipe(fp_uop_s);
13269 %}
13270
13271 instruct absD_reg(vRegD dst, vRegD src) %{
13272 match(Set dst (AbsD src));
13273
13274 ins_cost(INSN_COST * 3);
13275 format %{ "fabsd $dst, $src" %}
13276 ins_encode %{
13277 __ fabsd(as_FloatRegister($dst$$reg),
13278 as_FloatRegister($src$$reg));
13279 %}
13280
13281 ins_pipe(fp_uop_d);
13282 %}
13283
13284 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13285 match(Set dst (AbsF (SubF src1 src2)));
13286
13287 ins_cost(INSN_COST * 3);
13288 format %{ "fabds $dst, $src1, $src2" %}
13289 ins_encode %{
13290 __ fabds(as_FloatRegister($dst$$reg),
13291 as_FloatRegister($src1$$reg),
13292 as_FloatRegister($src2$$reg));
13293 %}
13294
13295 ins_pipe(fp_uop_s);
13296 %}
13297
13298 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13299 match(Set dst (AbsD (SubD src1 src2)));
13300
13301 ins_cost(INSN_COST * 3);
13302 format %{ "fabdd $dst, $src1, $src2" %}
13303 ins_encode %{
13304 __ fabdd(as_FloatRegister($dst$$reg),
13305 as_FloatRegister($src1$$reg),
13306 as_FloatRegister($src2$$reg));
13307 %}
13308
13309 ins_pipe(fp_uop_d);
13310 %}
13311
13312 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13313 match(Set dst (SqrtD src));
13314
13315 ins_cost(INSN_COST * 50);
13316 format %{ "fsqrtd $dst, $src" %}
13317 ins_encode %{
13318 __ fsqrtd(as_FloatRegister($dst$$reg),
13319 as_FloatRegister($src$$reg));
13320 %}
13321
13322 ins_pipe(fp_div_s);
13323 %}
13324
13325 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13326 match(Set dst (SqrtF src));
13327
13328 ins_cost(INSN_COST * 50);
13329 format %{ "fsqrts $dst, $src" %}
13330 ins_encode %{
13331 __ fsqrts(as_FloatRegister($dst$$reg),
13332 as_FloatRegister($src$$reg));
13333 %}
13334
13335 ins_pipe(fp_div_d);
13336 %}
13337
13338 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13339 match(Set dst (SqrtHF src));
13340 format %{ "fsqrth $dst, $src" %}
13341 ins_encode %{
13342 __ fsqrth($dst$$FloatRegister,
13343 $src$$FloatRegister);
13344 %}
13345 ins_pipe(fp_div_s);
13346 %}
13347
13348 // Math.rint, floor, ceil
13349 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13350 match(Set dst (RoundDoubleMode src rmode));
13351 format %{ "frint $dst, $src, $rmode" %}
13352 ins_encode %{
13353 switch ($rmode$$constant) {
13354 case RoundDoubleModeNode::rmode_rint:
13355 __ frintnd(as_FloatRegister($dst$$reg),
13356 as_FloatRegister($src$$reg));
13357 break;
13358 case RoundDoubleModeNode::rmode_floor:
13359 __ frintmd(as_FloatRegister($dst$$reg),
13360 as_FloatRegister($src$$reg));
13361 break;
13362 case RoundDoubleModeNode::rmode_ceil:
13363 __ frintpd(as_FloatRegister($dst$$reg),
13364 as_FloatRegister($src$$reg));
13365 break;
13366 }
13367 %}
13368 ins_pipe(fp_uop_d);
13369 %}
13370
13371 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13372 match(Set dst (CopySignD src1 (Binary src2 zero)));
13373 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13374 format %{ "CopySignD $dst $src1 $src2" %}
13375 ins_encode %{
13376 FloatRegister dst = as_FloatRegister($dst$$reg),
13377 src1 = as_FloatRegister($src1$$reg),
13378 src2 = as_FloatRegister($src2$$reg),
13379 zero = as_FloatRegister($zero$$reg);
13380 __ fnegd(dst, zero);
13381 __ bsl(dst, __ T8B, src2, src1);
13382 %}
13383 ins_pipe(fp_uop_d);
13384 %}
13385
13386 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13387 match(Set dst (CopySignF src1 src2));
13388 effect(TEMP_DEF dst, USE src1, USE src2);
13389 format %{ "CopySignF $dst $src1 $src2" %}
13390 ins_encode %{
13391 FloatRegister dst = as_FloatRegister($dst$$reg),
13392 src1 = as_FloatRegister($src1$$reg),
13393 src2 = as_FloatRegister($src2$$reg);
13394 __ movi(dst, __ T2S, 0x80, 24);
13395 __ bsl(dst, __ T8B, src2, src1);
13396 %}
13397 ins_pipe(fp_uop_d);
13398 %}
13399
13400 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13401 match(Set dst (SignumD src (Binary zero one)));
13402 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13403 format %{ "signumD $dst, $src" %}
13404 ins_encode %{
13405 FloatRegister src = as_FloatRegister($src$$reg),
13406 dst = as_FloatRegister($dst$$reg),
13407 zero = as_FloatRegister($zero$$reg),
13408 one = as_FloatRegister($one$$reg);
13409 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13410 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13411 // Bit selection instruction gets bit from "one" for each enabled bit in
13412 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13413 // NaN the whole "src" will be copied because "dst" is zero. For all other
13414 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13415 // from "src", and all other bits are copied from 1.0.
13416 __ bsl(dst, __ T8B, one, src);
13417 %}
13418 ins_pipe(fp_uop_d);
13419 %}
13420
13421 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13422 match(Set dst (SignumF src (Binary zero one)));
13423 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13424 format %{ "signumF $dst, $src" %}
13425 ins_encode %{
13426 FloatRegister src = as_FloatRegister($src$$reg),
13427 dst = as_FloatRegister($dst$$reg),
13428 zero = as_FloatRegister($zero$$reg),
13429 one = as_FloatRegister($one$$reg);
13430 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13431 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13432 // Bit selection instruction gets bit from "one" for each enabled bit in
13433 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13434 // NaN the whole "src" will be copied because "dst" is zero. For all other
13435 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13436 // from "src", and all other bits are copied from 1.0.
13437 __ bsl(dst, __ T8B, one, src);
13438 %}
13439 ins_pipe(fp_uop_d);
13440 %}
13441
13442 instruct onspinwait() %{
13443 match(OnSpinWait);
13444 ins_cost(INSN_COST);
13445
13446 format %{ "onspinwait" %}
13447
13448 ins_encode %{
13449 __ spin_wait();
13450 %}
13451 ins_pipe(pipe_class_empty);
13452 %}
13453
13454 // ============================================================================
13455 // Logical Instructions
13456
13457 // Integer Logical Instructions
13458
13459 // And Instructions
13460
13461
13462 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13463 match(Set dst (AndI src1 src2));
13464
13465 format %{ "andw $dst, $src1, $src2\t# int" %}
13466
13467 ins_cost(INSN_COST);
13468 ins_encode %{
13469 __ andw(as_Register($dst$$reg),
13470 as_Register($src1$$reg),
13471 as_Register($src2$$reg));
13472 %}
13473
13474 ins_pipe(ialu_reg_reg);
13475 %}
13476
13477 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13478 match(Set dst (AndI src1 src2));
13479
13480 format %{ "andsw $dst, $src1, $src2\t# int" %}
13481
13482 ins_cost(INSN_COST);
13483 ins_encode %{
13484 __ andw(as_Register($dst$$reg),
13485 as_Register($src1$$reg),
13486 (uint64_t)($src2$$constant));
13487 %}
13488
13489 ins_pipe(ialu_reg_imm);
13490 %}
13491
13492 // Or Instructions
13493
13494 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13495 match(Set dst (OrI src1 src2));
13496
13497 format %{ "orrw $dst, $src1, $src2\t# int" %}
13498
13499 ins_cost(INSN_COST);
13500 ins_encode %{
13501 __ orrw(as_Register($dst$$reg),
13502 as_Register($src1$$reg),
13503 as_Register($src2$$reg));
13504 %}
13505
13506 ins_pipe(ialu_reg_reg);
13507 %}
13508
13509 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13510 match(Set dst (OrI src1 src2));
13511
13512 format %{ "orrw $dst, $src1, $src2\t# int" %}
13513
13514 ins_cost(INSN_COST);
13515 ins_encode %{
13516 __ orrw(as_Register($dst$$reg),
13517 as_Register($src1$$reg),
13518 (uint64_t)($src2$$constant));
13519 %}
13520
13521 ins_pipe(ialu_reg_imm);
13522 %}
13523
13524 // Xor Instructions
13525
13526 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13527 match(Set dst (XorI src1 src2));
13528
13529 format %{ "eorw $dst, $src1, $src2\t# int" %}
13530
13531 ins_cost(INSN_COST);
13532 ins_encode %{
13533 __ eorw(as_Register($dst$$reg),
13534 as_Register($src1$$reg),
13535 as_Register($src2$$reg));
13536 %}
13537
13538 ins_pipe(ialu_reg_reg);
13539 %}
13540
13541 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13542 match(Set dst (XorI src1 src2));
13543
13544 format %{ "eorw $dst, $src1, $src2\t# int" %}
13545
13546 ins_cost(INSN_COST);
13547 ins_encode %{
13548 __ eorw(as_Register($dst$$reg),
13549 as_Register($src1$$reg),
13550 (uint64_t)($src2$$constant));
13551 %}
13552
13553 ins_pipe(ialu_reg_imm);
13554 %}
13555
13556 // Long Logical Instructions
13557 // TODO
13558
13559 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13560 match(Set dst (AndL src1 src2));
13561
13562 format %{ "and $dst, $src1, $src2\t# int" %}
13563
13564 ins_cost(INSN_COST);
13565 ins_encode %{
13566 __ andr(as_Register($dst$$reg),
13567 as_Register($src1$$reg),
13568 as_Register($src2$$reg));
13569 %}
13570
13571 ins_pipe(ialu_reg_reg);
13572 %}
13573
13574 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13575 match(Set dst (AndL src1 src2));
13576
13577 format %{ "and $dst, $src1, $src2\t# int" %}
13578
13579 ins_cost(INSN_COST);
13580 ins_encode %{
13581 __ andr(as_Register($dst$$reg),
13582 as_Register($src1$$reg),
13583 (uint64_t)($src2$$constant));
13584 %}
13585
13586 ins_pipe(ialu_reg_imm);
13587 %}
13588
13589 // Or Instructions
13590
13591 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13592 match(Set dst (OrL src1 src2));
13593
13594 format %{ "orr $dst, $src1, $src2\t# int" %}
13595
13596 ins_cost(INSN_COST);
13597 ins_encode %{
13598 __ orr(as_Register($dst$$reg),
13599 as_Register($src1$$reg),
13600 as_Register($src2$$reg));
13601 %}
13602
13603 ins_pipe(ialu_reg_reg);
13604 %}
13605
13606 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13607 match(Set dst (OrL src1 src2));
13608
13609 format %{ "orr $dst, $src1, $src2\t# int" %}
13610
13611 ins_cost(INSN_COST);
13612 ins_encode %{
13613 __ orr(as_Register($dst$$reg),
13614 as_Register($src1$$reg),
13615 (uint64_t)($src2$$constant));
13616 %}
13617
13618 ins_pipe(ialu_reg_imm);
13619 %}
13620
13621 // Xor Instructions
13622
13623 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13624 match(Set dst (XorL src1 src2));
13625
13626 format %{ "eor $dst, $src1, $src2\t# int" %}
13627
13628 ins_cost(INSN_COST);
13629 ins_encode %{
13630 __ eor(as_Register($dst$$reg),
13631 as_Register($src1$$reg),
13632 as_Register($src2$$reg));
13633 %}
13634
13635 ins_pipe(ialu_reg_reg);
13636 %}
13637
13638 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13639 match(Set dst (XorL src1 src2));
13640
13641 ins_cost(INSN_COST);
13642 format %{ "eor $dst, $src1, $src2\t# int" %}
13643
13644 ins_encode %{
13645 __ eor(as_Register($dst$$reg),
13646 as_Register($src1$$reg),
13647 (uint64_t)($src2$$constant));
13648 %}
13649
13650 ins_pipe(ialu_reg_imm);
13651 %}
13652
13653 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13654 %{
13655 match(Set dst (ConvI2L src));
13656
13657 ins_cost(INSN_COST);
13658 format %{ "sxtw $dst, $src\t# i2l" %}
13659 ins_encode %{
13660 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13661 %}
13662 ins_pipe(ialu_reg_shift);
13663 %}
13664
13665 // this pattern occurs in bigmath arithmetic
13666 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13667 %{
13668 match(Set dst (AndL (ConvI2L src) mask));
13669
13670 ins_cost(INSN_COST);
13671 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13672 ins_encode %{
13673 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13674 %}
13675
13676 ins_pipe(ialu_reg_shift);
13677 %}
13678
13679 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13680 match(Set dst (ConvL2I src));
13681
13682 ins_cost(INSN_COST);
13683 format %{ "movw $dst, $src \t// l2i" %}
13684
13685 ins_encode %{
13686 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13687 %}
13688
13689 ins_pipe(ialu_reg);
13690 %}
13691
13692 instruct convD2F_reg(vRegF dst, vRegD src) %{
13693 match(Set dst (ConvD2F src));
13694
13695 ins_cost(INSN_COST * 5);
13696 format %{ "fcvtd $dst, $src \t// d2f" %}
13697
13698 ins_encode %{
13699 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13700 %}
13701
13702 ins_pipe(fp_d2f);
13703 %}
13704
13705 instruct convF2D_reg(vRegD dst, vRegF src) %{
13706 match(Set dst (ConvF2D src));
13707
13708 ins_cost(INSN_COST * 5);
13709 format %{ "fcvts $dst, $src \t// f2d" %}
13710
13711 ins_encode %{
13712 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13713 %}
13714
13715 ins_pipe(fp_f2d);
13716 %}
13717
13718 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13719 match(Set dst (ConvF2I src));
13720
13721 ins_cost(INSN_COST * 5);
13722 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13723
13724 ins_encode %{
13725 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13726 %}
13727
13728 ins_pipe(fp_f2i);
13729 %}
13730
13731 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13732 match(Set dst (ConvF2L src));
13733
13734 ins_cost(INSN_COST * 5);
13735 format %{ "fcvtzs $dst, $src \t// f2l" %}
13736
13737 ins_encode %{
13738 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13739 %}
13740
13741 ins_pipe(fp_f2l);
13742 %}
13743
13744 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13745 match(Set dst (ConvF2HF src));
13746 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13747 "smov $dst, $tmp\t# move result from $tmp to $dst"
13748 %}
13749 effect(TEMP tmp);
13750 ins_encode %{
13751 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13752 %}
13753 ins_pipe(pipe_slow);
13754 %}
13755
13756 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13757 match(Set dst (ConvHF2F src));
13758 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13759 "fcvt $dst, $tmp\t# convert half to single precision"
13760 %}
13761 effect(TEMP tmp);
13762 ins_encode %{
13763 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13764 %}
13765 ins_pipe(pipe_slow);
13766 %}
13767
13768 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13769 match(Set dst (ConvI2F src));
13770
13771 ins_cost(INSN_COST * 5);
13772 format %{ "scvtfws $dst, $src \t// i2f" %}
13773
13774 ins_encode %{
13775 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13776 %}
13777
13778 ins_pipe(fp_i2f);
13779 %}
13780
13781 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13782 match(Set dst (ConvL2F src));
13783
13784 ins_cost(INSN_COST * 5);
13785 format %{ "scvtfs $dst, $src \t// l2f" %}
13786
13787 ins_encode %{
13788 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13789 %}
13790
13791 ins_pipe(fp_l2f);
13792 %}
13793
13794 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13795 match(Set dst (ConvD2I src));
13796
13797 ins_cost(INSN_COST * 5);
13798 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13799
13800 ins_encode %{
13801 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13802 %}
13803
13804 ins_pipe(fp_d2i);
13805 %}
13806
13807 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13808 match(Set dst (ConvD2L src));
13809
13810 ins_cost(INSN_COST * 5);
13811 format %{ "fcvtzd $dst, $src \t// d2l" %}
13812
13813 ins_encode %{
13814 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13815 %}
13816
13817 ins_pipe(fp_d2l);
13818 %}
13819
13820 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13821 match(Set dst (ConvI2D src));
13822
13823 ins_cost(INSN_COST * 5);
13824 format %{ "scvtfwd $dst, $src \t// i2d" %}
13825
13826 ins_encode %{
13827 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13828 %}
13829
13830 ins_pipe(fp_i2d);
13831 %}
13832
13833 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13834 match(Set dst (ConvL2D src));
13835
13836 ins_cost(INSN_COST * 5);
13837 format %{ "scvtfd $dst, $src \t// l2d" %}
13838
13839 ins_encode %{
13840 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13841 %}
13842
13843 ins_pipe(fp_l2d);
13844 %}
13845
13846 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13847 %{
13848 match(Set dst (RoundD src));
13849 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13850 format %{ "java_round_double $dst,$src"%}
13851 ins_encode %{
13852 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13853 as_FloatRegister($ftmp$$reg));
13854 %}
13855 ins_pipe(pipe_slow);
13856 %}
13857
13858 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13859 %{
13860 match(Set dst (RoundF src));
13861 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13862 format %{ "java_round_float $dst,$src"%}
13863 ins_encode %{
13864 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13865 as_FloatRegister($ftmp$$reg));
13866 %}
13867 ins_pipe(pipe_slow);
13868 %}
13869
13870 // stack <-> reg and reg <-> reg shuffles with no conversion
13871
13872 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13873
13874 match(Set dst (MoveF2I src));
13875
13876 effect(DEF dst, USE src);
13877
13878 ins_cost(4 * INSN_COST);
13879
13880 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13881
13882 ins_encode %{
13883 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13884 %}
13885
13886 ins_pipe(iload_reg_reg);
13887
13888 %}
13889
13890 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13891
13892 match(Set dst (MoveI2F src));
13893
13894 effect(DEF dst, USE src);
13895
13896 ins_cost(4 * INSN_COST);
13897
13898 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13899
13900 ins_encode %{
13901 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13902 %}
13903
13904 ins_pipe(pipe_class_memory);
13905
13906 %}
13907
13908 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13909
13910 match(Set dst (MoveD2L src));
13911
13912 effect(DEF dst, USE src);
13913
13914 ins_cost(4 * INSN_COST);
13915
13916 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13917
13918 ins_encode %{
13919 __ ldr($dst$$Register, Address(sp, $src$$disp));
13920 %}
13921
13922 ins_pipe(iload_reg_reg);
13923
13924 %}
13925
13926 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13927
13928 match(Set dst (MoveL2D src));
13929
13930 effect(DEF dst, USE src);
13931
13932 ins_cost(4 * INSN_COST);
13933
13934 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13935
13936 ins_encode %{
13937 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13938 %}
13939
13940 ins_pipe(pipe_class_memory);
13941
13942 %}
13943
13944 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13945
13946 match(Set dst (MoveF2I src));
13947
13948 effect(DEF dst, USE src);
13949
13950 ins_cost(INSN_COST);
13951
13952 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13953
13954 ins_encode %{
13955 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13956 %}
13957
13958 ins_pipe(pipe_class_memory);
13959
13960 %}
13961
13962 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13963
13964 match(Set dst (MoveI2F src));
13965
13966 effect(DEF dst, USE src);
13967
13968 ins_cost(INSN_COST);
13969
13970 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13971
13972 ins_encode %{
13973 __ strw($src$$Register, Address(sp, $dst$$disp));
13974 %}
13975
13976 ins_pipe(istore_reg_reg);
13977
13978 %}
13979
13980 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13981
13982 match(Set dst (MoveD2L src));
13983
13984 effect(DEF dst, USE src);
13985
13986 ins_cost(INSN_COST);
13987
13988 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13989
13990 ins_encode %{
13991 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13992 %}
13993
13994 ins_pipe(pipe_class_memory);
13995
13996 %}
13997
13998 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13999
14000 match(Set dst (MoveL2D src));
14001
14002 effect(DEF dst, USE src);
14003
14004 ins_cost(INSN_COST);
14005
14006 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14007
14008 ins_encode %{
14009 __ str($src$$Register, Address(sp, $dst$$disp));
14010 %}
14011
14012 ins_pipe(istore_reg_reg);
14013
14014 %}
14015
14016 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14017
14018 match(Set dst (MoveF2I src));
14019
14020 effect(DEF dst, USE src);
14021
14022 ins_cost(INSN_COST);
14023
14024 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14025
14026 ins_encode %{
14027 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14028 %}
14029
14030 ins_pipe(fp_f2i);
14031
14032 %}
14033
14034 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14035
14036 match(Set dst (MoveI2F src));
14037
14038 effect(DEF dst, USE src);
14039
14040 ins_cost(INSN_COST);
14041
14042 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14043
14044 ins_encode %{
14045 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14046 %}
14047
14048 ins_pipe(fp_i2f);
14049
14050 %}
14051
14052 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14053
14054 match(Set dst (MoveD2L src));
14055
14056 effect(DEF dst, USE src);
14057
14058 ins_cost(INSN_COST);
14059
14060 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14061
14062 ins_encode %{
14063 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14064 %}
14065
14066 ins_pipe(fp_d2l);
14067
14068 %}
14069
14070 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14071
14072 match(Set dst (MoveL2D src));
14073
14074 effect(DEF dst, USE src);
14075
14076 ins_cost(INSN_COST);
14077
14078 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14079
14080 ins_encode %{
14081 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14082 %}
14083
14084 ins_pipe(fp_l2d);
14085
14086 %}
14087
14088 // ============================================================================
14089 // clearing of an array
14090
14091 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14092 %{
14093 match(Set dummy (ClearArray cnt base));
14094 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14095
14096 ins_cost(4 * INSN_COST);
14097 format %{ "ClearArray $cnt, $base" %}
14098
14099 ins_encode %{
14100 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14101 if (tpc == nullptr) {
14102 ciEnv::current()->record_failure("CodeCache is full");
14103 return;
14104 }
14105 %}
14106
14107 ins_pipe(pipe_class_memory);
14108 %}
14109
14110 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14111 %{
14112 predicate((uint64_t)n->in(2)->get_long()
14113 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14114 match(Set dummy (ClearArray cnt base));
14115 effect(TEMP temp, USE_KILL base, KILL cr);
14116
14117 ins_cost(4 * INSN_COST);
14118 format %{ "ClearArray $cnt, $base" %}
14119
14120 ins_encode %{
14121 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14122 if (tpc == nullptr) {
14123 ciEnv::current()->record_failure("CodeCache is full");
14124 return;
14125 }
14126 %}
14127
14128 ins_pipe(pipe_class_memory);
14129 %}
14130
14131 // ============================================================================
14132 // Overflow Math Instructions
14133
14134 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14135 %{
14136 match(Set cr (OverflowAddI op1 op2));
14137
14138 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14139 ins_cost(INSN_COST);
14140 ins_encode %{
14141 __ cmnw($op1$$Register, $op2$$Register);
14142 %}
14143
14144 ins_pipe(icmp_reg_reg);
14145 %}
14146
14147 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14148 %{
14149 match(Set cr (OverflowAddI op1 op2));
14150
14151 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14152 ins_cost(INSN_COST);
14153 ins_encode %{
14154 __ cmnw($op1$$Register, $op2$$constant);
14155 %}
14156
14157 ins_pipe(icmp_reg_imm);
14158 %}
14159
14160 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14161 %{
14162 match(Set cr (OverflowAddL op1 op2));
14163
14164 format %{ "cmn $op1, $op2\t# overflow check long" %}
14165 ins_cost(INSN_COST);
14166 ins_encode %{
14167 __ cmn($op1$$Register, $op2$$Register);
14168 %}
14169
14170 ins_pipe(icmp_reg_reg);
14171 %}
14172
14173 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14174 %{
14175 match(Set cr (OverflowAddL op1 op2));
14176
14177 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14178 ins_cost(INSN_COST);
14179 ins_encode %{
14180 __ adds(zr, $op1$$Register, $op2$$constant);
14181 %}
14182
14183 ins_pipe(icmp_reg_imm);
14184 %}
14185
14186 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14187 %{
14188 match(Set cr (OverflowSubI op1 op2));
14189
14190 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14191 ins_cost(INSN_COST);
14192 ins_encode %{
14193 __ cmpw($op1$$Register, $op2$$Register);
14194 %}
14195
14196 ins_pipe(icmp_reg_reg);
14197 %}
14198
14199 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14200 %{
14201 match(Set cr (OverflowSubI op1 op2));
14202
14203 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14204 ins_cost(INSN_COST);
14205 ins_encode %{
14206 __ cmpw($op1$$Register, $op2$$constant);
14207 %}
14208
14209 ins_pipe(icmp_reg_imm);
14210 %}
14211
14212 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14213 %{
14214 match(Set cr (OverflowSubL op1 op2));
14215
14216 format %{ "cmp $op1, $op2\t# overflow check long" %}
14217 ins_cost(INSN_COST);
14218 ins_encode %{
14219 __ cmp($op1$$Register, $op2$$Register);
14220 %}
14221
14222 ins_pipe(icmp_reg_reg);
14223 %}
14224
14225 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14226 %{
14227 match(Set cr (OverflowSubL op1 op2));
14228
14229 format %{ "cmp $op1, $op2\t# overflow check long" %}
14230 ins_cost(INSN_COST);
14231 ins_encode %{
14232 __ subs(zr, $op1$$Register, $op2$$constant);
14233 %}
14234
14235 ins_pipe(icmp_reg_imm);
14236 %}
14237
14238 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14239 %{
14240 match(Set cr (OverflowSubI zero op1));
14241
14242 format %{ "cmpw zr, $op1\t# overflow check int" %}
14243 ins_cost(INSN_COST);
14244 ins_encode %{
14245 __ cmpw(zr, $op1$$Register);
14246 %}
14247
14248 ins_pipe(icmp_reg_imm);
14249 %}
14250
14251 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14252 %{
14253 match(Set cr (OverflowSubL zero op1));
14254
14255 format %{ "cmp zr, $op1\t# overflow check long" %}
14256 ins_cost(INSN_COST);
14257 ins_encode %{
14258 __ cmp(zr, $op1$$Register);
14259 %}
14260
14261 ins_pipe(icmp_reg_imm);
14262 %}
14263
14264 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14265 %{
14266 match(Set cr (OverflowMulI op1 op2));
14267
14268 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14269 "cmp rscratch1, rscratch1, sxtw\n\t"
14270 "movw rscratch1, #0x80000000\n\t"
14271 "cselw rscratch1, rscratch1, zr, NE\n\t"
14272 "cmpw rscratch1, #1" %}
14273 ins_cost(5 * INSN_COST);
14274 ins_encode %{
14275 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14276 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14277 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14278 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14279 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14280 %}
14281
14282 ins_pipe(pipe_slow);
14283 %}
14284
14285 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14286 %{
14287 match(If cmp (OverflowMulI op1 op2));
14288 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14289 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14290 effect(USE labl, KILL cr);
14291
14292 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14293 "cmp rscratch1, rscratch1, sxtw\n\t"
14294 "b$cmp $labl" %}
14295 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14296 ins_encode %{
14297 Label* L = $labl$$label;
14298 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14299 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14300 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14301 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14302 %}
14303
14304 ins_pipe(pipe_serial);
14305 %}
14306
14307 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14308 %{
14309 match(Set cr (OverflowMulL op1 op2));
14310
14311 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14312 "smulh rscratch2, $op1, $op2\n\t"
14313 "cmp rscratch2, rscratch1, ASR #63\n\t"
14314 "movw rscratch1, #0x80000000\n\t"
14315 "cselw rscratch1, rscratch1, zr, NE\n\t"
14316 "cmpw rscratch1, #1" %}
14317 ins_cost(6 * INSN_COST);
14318 ins_encode %{
14319 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14320 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14321 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14322 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14323 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14324 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14325 %}
14326
14327 ins_pipe(pipe_slow);
14328 %}
14329
14330 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14331 %{
14332 match(If cmp (OverflowMulL op1 op2));
14333 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14334 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14335 effect(USE labl, KILL cr);
14336
14337 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14338 "smulh rscratch2, $op1, $op2\n\t"
14339 "cmp rscratch2, rscratch1, ASR #63\n\t"
14340 "b$cmp $labl" %}
14341 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14342 ins_encode %{
14343 Label* L = $labl$$label;
14344 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14345 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14346 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14347 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14348 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14349 %}
14350
14351 ins_pipe(pipe_serial);
14352 %}
14353
14354 // ============================================================================
14355 // Compare Instructions
14356
14357 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14358 %{
14359 match(Set cr (CmpI op1 op2));
14360
14361 effect(DEF cr, USE op1, USE op2);
14362
14363 ins_cost(INSN_COST);
14364 format %{ "cmpw $op1, $op2" %}
14365
14366 ins_encode(aarch64_enc_cmpw(op1, op2));
14367
14368 ins_pipe(icmp_reg_reg);
14369 %}
14370
14371 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14372 %{
14373 match(Set cr (CmpI op1 zero));
14374
14375 effect(DEF cr, USE op1);
14376
14377 ins_cost(INSN_COST);
14378 format %{ "cmpw $op1, 0" %}
14379
14380 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14381
14382 ins_pipe(icmp_reg_imm);
14383 %}
14384
14385 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14386 %{
14387 match(Set cr (CmpI op1 op2));
14388
14389 effect(DEF cr, USE op1);
14390
14391 ins_cost(INSN_COST);
14392 format %{ "cmpw $op1, $op2" %}
14393
14394 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14395
14396 ins_pipe(icmp_reg_imm);
14397 %}
14398
14399 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14400 %{
14401 match(Set cr (CmpI op1 op2));
14402
14403 effect(DEF cr, USE op1);
14404
14405 ins_cost(INSN_COST * 2);
14406 format %{ "cmpw $op1, $op2" %}
14407
14408 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14409
14410 ins_pipe(icmp_reg_imm);
14411 %}
14412
14413 // Unsigned compare Instructions; really, same as signed compare
14414 // except it should only be used to feed an If or a CMovI which takes a
14415 // cmpOpU.
14416
14417 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14418 %{
14419 match(Set cr (CmpU op1 op2));
14420
14421 effect(DEF cr, USE op1, USE op2);
14422
14423 ins_cost(INSN_COST);
14424 format %{ "cmpw $op1, $op2\t# unsigned" %}
14425
14426 ins_encode(aarch64_enc_cmpw(op1, op2));
14427
14428 ins_pipe(icmp_reg_reg);
14429 %}
14430
14431 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14432 %{
14433 match(Set cr (CmpU op1 zero));
14434
14435 effect(DEF cr, USE op1);
14436
14437 ins_cost(INSN_COST);
14438 format %{ "cmpw $op1, #0\t# unsigned" %}
14439
14440 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14441
14442 ins_pipe(icmp_reg_imm);
14443 %}
14444
14445 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14446 %{
14447 match(Set cr (CmpU op1 op2));
14448
14449 effect(DEF cr, USE op1);
14450
14451 ins_cost(INSN_COST);
14452 format %{ "cmpw $op1, $op2\t# unsigned" %}
14453
14454 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14455
14456 ins_pipe(icmp_reg_imm);
14457 %}
14458
14459 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14460 %{
14461 match(Set cr (CmpU op1 op2));
14462
14463 effect(DEF cr, USE op1);
14464
14465 ins_cost(INSN_COST * 2);
14466 format %{ "cmpw $op1, $op2\t# unsigned" %}
14467
14468 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14469
14470 ins_pipe(icmp_reg_imm);
14471 %}
14472
14473 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14474 %{
14475 match(Set cr (CmpL op1 op2));
14476
14477 effect(DEF cr, USE op1, USE op2);
14478
14479 ins_cost(INSN_COST);
14480 format %{ "cmp $op1, $op2" %}
14481
14482 ins_encode(aarch64_enc_cmp(op1, op2));
14483
14484 ins_pipe(icmp_reg_reg);
14485 %}
14486
14487 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14488 %{
14489 match(Set cr (CmpL op1 zero));
14490
14491 effect(DEF cr, USE op1);
14492
14493 ins_cost(INSN_COST);
14494 format %{ "tst $op1" %}
14495
14496 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14497
14498 ins_pipe(icmp_reg_imm);
14499 %}
14500
14501 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14502 %{
14503 match(Set cr (CmpL op1 op2));
14504
14505 effect(DEF cr, USE op1);
14506
14507 ins_cost(INSN_COST);
14508 format %{ "cmp $op1, $op2" %}
14509
14510 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14511
14512 ins_pipe(icmp_reg_imm);
14513 %}
14514
14515 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14516 %{
14517 match(Set cr (CmpL op1 op2));
14518
14519 effect(DEF cr, USE op1);
14520
14521 ins_cost(INSN_COST * 2);
14522 format %{ "cmp $op1, $op2" %}
14523
14524 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14525
14526 ins_pipe(icmp_reg_imm);
14527 %}
14528
14529 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14530 %{
14531 match(Set cr (CmpUL op1 op2));
14532
14533 effect(DEF cr, USE op1, USE op2);
14534
14535 ins_cost(INSN_COST);
14536 format %{ "cmp $op1, $op2" %}
14537
14538 ins_encode(aarch64_enc_cmp(op1, op2));
14539
14540 ins_pipe(icmp_reg_reg);
14541 %}
14542
14543 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14544 %{
14545 match(Set cr (CmpUL op1 zero));
14546
14547 effect(DEF cr, USE op1);
14548
14549 ins_cost(INSN_COST);
14550 format %{ "tst $op1" %}
14551
14552 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14553
14554 ins_pipe(icmp_reg_imm);
14555 %}
14556
14557 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14558 %{
14559 match(Set cr (CmpUL op1 op2));
14560
14561 effect(DEF cr, USE op1);
14562
14563 ins_cost(INSN_COST);
14564 format %{ "cmp $op1, $op2" %}
14565
14566 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14567
14568 ins_pipe(icmp_reg_imm);
14569 %}
14570
14571 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14572 %{
14573 match(Set cr (CmpUL op1 op2));
14574
14575 effect(DEF cr, USE op1);
14576
14577 ins_cost(INSN_COST * 2);
14578 format %{ "cmp $op1, $op2" %}
14579
14580 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14581
14582 ins_pipe(icmp_reg_imm);
14583 %}
14584
14585 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14586 %{
14587 match(Set cr (CmpP op1 op2));
14588
14589 effect(DEF cr, USE op1, USE op2);
14590
14591 ins_cost(INSN_COST);
14592 format %{ "cmp $op1, $op2\t // ptr" %}
14593
14594 ins_encode(aarch64_enc_cmpp(op1, op2));
14595
14596 ins_pipe(icmp_reg_reg);
14597 %}
14598
14599 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14600 %{
14601 match(Set cr (CmpN op1 op2));
14602
14603 effect(DEF cr, USE op1, USE op2);
14604
14605 ins_cost(INSN_COST);
14606 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14607
14608 ins_encode(aarch64_enc_cmpn(op1, op2));
14609
14610 ins_pipe(icmp_reg_reg);
14611 %}
14612
14613 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14614 %{
14615 match(Set cr (CmpP op1 zero));
14616
14617 effect(DEF cr, USE op1, USE zero);
14618
14619 ins_cost(INSN_COST);
14620 format %{ "cmp $op1, 0\t // ptr" %}
14621
14622 ins_encode(aarch64_enc_testp(op1));
14623
14624 ins_pipe(icmp_reg_imm);
14625 %}
14626
14627 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14628 %{
14629 match(Set cr (CmpN op1 zero));
14630
14631 effect(DEF cr, USE op1, USE zero);
14632
14633 ins_cost(INSN_COST);
14634 format %{ "cmp $op1, 0\t // compressed ptr" %}
14635
14636 ins_encode(aarch64_enc_testn(op1));
14637
14638 ins_pipe(icmp_reg_imm);
14639 %}
14640
14641 // FP comparisons
14642 //
14643 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14644 // using normal cmpOp. See declaration of rFlagsReg for details.
14645
14646 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14647 %{
14648 match(Set cr (CmpF src1 src2));
14649
14650 ins_cost(3 * INSN_COST);
14651 format %{ "fcmps $src1, $src2" %}
14652
14653 ins_encode %{
14654 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14655 %}
14656
14657 ins_pipe(pipe_class_compare);
14658 %}
14659
14660 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14661 %{
14662 match(Set cr (CmpF src1 src2));
14663
14664 ins_cost(3 * INSN_COST);
14665 format %{ "fcmps $src1, 0.0" %}
14666
14667 ins_encode %{
14668 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14669 %}
14670
14671 ins_pipe(pipe_class_compare);
14672 %}
14673 // FROM HERE
14674
14675 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14676 %{
14677 match(Set cr (CmpD src1 src2));
14678
14679 ins_cost(3 * INSN_COST);
14680 format %{ "fcmpd $src1, $src2" %}
14681
14682 ins_encode %{
14683 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14684 %}
14685
14686 ins_pipe(pipe_class_compare);
14687 %}
14688
14689 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14690 %{
14691 match(Set cr (CmpD src1 src2));
14692
14693 ins_cost(3 * INSN_COST);
14694 format %{ "fcmpd $src1, 0.0" %}
14695
14696 ins_encode %{
14697 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14698 %}
14699
14700 ins_pipe(pipe_class_compare);
14701 %}
14702
14703 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14704 %{
14705 match(Set dst (CmpF3 src1 src2));
14706 effect(KILL cr);
14707
14708 ins_cost(5 * INSN_COST);
14709 format %{ "fcmps $src1, $src2\n\t"
14710 "csinvw($dst, zr, zr, eq\n\t"
14711 "csnegw($dst, $dst, $dst, lt)"
14712 %}
14713
14714 ins_encode %{
14715 Label done;
14716 FloatRegister s1 = as_FloatRegister($src1$$reg);
14717 FloatRegister s2 = as_FloatRegister($src2$$reg);
14718 Register d = as_Register($dst$$reg);
14719 __ fcmps(s1, s2);
14720 // installs 0 if EQ else -1
14721 __ csinvw(d, zr, zr, Assembler::EQ);
14722 // keeps -1 if less or unordered else installs 1
14723 __ csnegw(d, d, d, Assembler::LT);
14724 __ bind(done);
14725 %}
14726
14727 ins_pipe(pipe_class_default);
14728
14729 %}
14730
14731 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14732 %{
14733 match(Set dst (CmpD3 src1 src2));
14734 effect(KILL cr);
14735
14736 ins_cost(5 * INSN_COST);
14737 format %{ "fcmpd $src1, $src2\n\t"
14738 "csinvw($dst, zr, zr, eq\n\t"
14739 "csnegw($dst, $dst, $dst, lt)"
14740 %}
14741
14742 ins_encode %{
14743 Label done;
14744 FloatRegister s1 = as_FloatRegister($src1$$reg);
14745 FloatRegister s2 = as_FloatRegister($src2$$reg);
14746 Register d = as_Register($dst$$reg);
14747 __ fcmpd(s1, s2);
14748 // installs 0 if EQ else -1
14749 __ csinvw(d, zr, zr, Assembler::EQ);
14750 // keeps -1 if less or unordered else installs 1
14751 __ csnegw(d, d, d, Assembler::LT);
14752 __ bind(done);
14753 %}
14754 ins_pipe(pipe_class_default);
14755
14756 %}
14757
14758 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14759 %{
14760 match(Set dst (CmpF3 src1 zero));
14761 effect(KILL cr);
14762
14763 ins_cost(5 * INSN_COST);
14764 format %{ "fcmps $src1, 0.0\n\t"
14765 "csinvw($dst, zr, zr, eq\n\t"
14766 "csnegw($dst, $dst, $dst, lt)"
14767 %}
14768
14769 ins_encode %{
14770 Label done;
14771 FloatRegister s1 = as_FloatRegister($src1$$reg);
14772 Register d = as_Register($dst$$reg);
14773 __ fcmps(s1, 0.0);
14774 // installs 0 if EQ else -1
14775 __ csinvw(d, zr, zr, Assembler::EQ);
14776 // keeps -1 if less or unordered else installs 1
14777 __ csnegw(d, d, d, Assembler::LT);
14778 __ bind(done);
14779 %}
14780
14781 ins_pipe(pipe_class_default);
14782
14783 %}
14784
14785 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14786 %{
14787 match(Set dst (CmpD3 src1 zero));
14788 effect(KILL cr);
14789
14790 ins_cost(5 * INSN_COST);
14791 format %{ "fcmpd $src1, 0.0\n\t"
14792 "csinvw($dst, zr, zr, eq\n\t"
14793 "csnegw($dst, $dst, $dst, lt)"
14794 %}
14795
14796 ins_encode %{
14797 Label done;
14798 FloatRegister s1 = as_FloatRegister($src1$$reg);
14799 Register d = as_Register($dst$$reg);
14800 __ fcmpd(s1, 0.0);
14801 // installs 0 if EQ else -1
14802 __ csinvw(d, zr, zr, Assembler::EQ);
14803 // keeps -1 if less or unordered else installs 1
14804 __ csnegw(d, d, d, Assembler::LT);
14805 __ bind(done);
14806 %}
14807 ins_pipe(pipe_class_default);
14808
14809 %}
14810
14811 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14812 %{
14813 match(Set dst (CmpLTMask p q));
14814 effect(KILL cr);
14815
14816 ins_cost(3 * INSN_COST);
14817
14818 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14819 "csetw $dst, lt\n\t"
14820 "subw $dst, zr, $dst"
14821 %}
14822
14823 ins_encode %{
14824 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14825 __ csetw(as_Register($dst$$reg), Assembler::LT);
14826 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14827 %}
14828
14829 ins_pipe(ialu_reg_reg);
14830 %}
14831
14832 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14833 %{
14834 match(Set dst (CmpLTMask src zero));
14835 effect(KILL cr);
14836
14837 ins_cost(INSN_COST);
14838
14839 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14840
14841 ins_encode %{
14842 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14843 %}
14844
14845 ins_pipe(ialu_reg_shift);
14846 %}
14847
14848 // ============================================================================
14849 // Max and Min
14850
14851 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14852
14853 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14854 %{
14855 effect(DEF cr, USE src);
14856 ins_cost(INSN_COST);
14857 format %{ "cmpw $src, 0" %}
14858
14859 ins_encode %{
14860 __ cmpw($src$$Register, 0);
14861 %}
14862 ins_pipe(icmp_reg_imm);
14863 %}
14864
14865 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14866 %{
14867 match(Set dst (MinI src1 src2));
14868 ins_cost(INSN_COST * 3);
14869
14870 expand %{
14871 rFlagsReg cr;
14872 compI_reg_reg(cr, src1, src2);
14873 cmovI_reg_reg_lt(dst, src1, src2, cr);
14874 %}
14875 %}
14876
14877 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14878 %{
14879 match(Set dst (MaxI src1 src2));
14880 ins_cost(INSN_COST * 3);
14881
14882 expand %{
14883 rFlagsReg cr;
14884 compI_reg_reg(cr, src1, src2);
14885 cmovI_reg_reg_gt(dst, src1, src2, cr);
14886 %}
14887 %}
14888
14889
14890 // ============================================================================
14891 // Branch Instructions
14892
14893 // Direct Branch.
14894 instruct branch(label lbl)
14895 %{
14896 match(Goto);
14897
14898 effect(USE lbl);
14899
14900 ins_cost(BRANCH_COST);
14901 format %{ "b $lbl" %}
14902
14903 ins_encode(aarch64_enc_b(lbl));
14904
14905 ins_pipe(pipe_branch);
14906 %}
14907
14908 // Conditional Near Branch
14909 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14910 %{
14911 // Same match rule as `branchConFar'.
14912 match(If cmp cr);
14913
14914 effect(USE lbl);
14915
14916 ins_cost(BRANCH_COST);
14917 // If set to 1 this indicates that the current instruction is a
14918 // short variant of a long branch. This avoids using this
14919 // instruction in first-pass matching. It will then only be used in
14920 // the `Shorten_branches' pass.
14921 // ins_short_branch(1);
14922 format %{ "b$cmp $lbl" %}
14923
14924 ins_encode(aarch64_enc_br_con(cmp, lbl));
14925
14926 ins_pipe(pipe_branch_cond);
14927 %}
14928
14929 // Conditional Near Branch Unsigned
14930 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14931 %{
14932 // Same match rule as `branchConFar'.
14933 match(If cmp cr);
14934
14935 effect(USE lbl);
14936
14937 ins_cost(BRANCH_COST);
14938 // If set to 1 this indicates that the current instruction is a
14939 // short variant of a long branch. This avoids using this
14940 // instruction in first-pass matching. It will then only be used in
14941 // the `Shorten_branches' pass.
14942 // ins_short_branch(1);
14943 format %{ "b$cmp $lbl\t# unsigned" %}
14944
14945 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14946
14947 ins_pipe(pipe_branch_cond);
14948 %}
14949
14950 // Make use of CBZ and CBNZ. These instructions, as well as being
14951 // shorter than (cmp; branch), have the additional benefit of not
14952 // killing the flags.
14953
14954 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14955 match(If cmp (CmpI op1 op2));
14956 effect(USE labl);
14957
14958 ins_cost(BRANCH_COST);
14959 format %{ "cbw$cmp $op1, $labl" %}
14960 ins_encode %{
14961 Label* L = $labl$$label;
14962 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14963 if (cond == Assembler::EQ)
14964 __ cbzw($op1$$Register, *L);
14965 else
14966 __ cbnzw($op1$$Register, *L);
14967 %}
14968 ins_pipe(pipe_cmp_branch);
14969 %}
14970
14971 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14972 match(If cmp (CmpL op1 op2));
14973 effect(USE labl);
14974
14975 ins_cost(BRANCH_COST);
14976 format %{ "cb$cmp $op1, $labl" %}
14977 ins_encode %{
14978 Label* L = $labl$$label;
14979 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14980 if (cond == Assembler::EQ)
14981 __ cbz($op1$$Register, *L);
14982 else
14983 __ cbnz($op1$$Register, *L);
14984 %}
14985 ins_pipe(pipe_cmp_branch);
14986 %}
14987
14988 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14989 match(If cmp (CmpP op1 op2));
14990 effect(USE labl);
14991
14992 ins_cost(BRANCH_COST);
14993 format %{ "cb$cmp $op1, $labl" %}
14994 ins_encode %{
14995 Label* L = $labl$$label;
14996 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14997 if (cond == Assembler::EQ)
14998 __ cbz($op1$$Register, *L);
14999 else
15000 __ cbnz($op1$$Register, *L);
15001 %}
15002 ins_pipe(pipe_cmp_branch);
15003 %}
15004
15005 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15006 match(If cmp (CmpN op1 op2));
15007 effect(USE labl);
15008
15009 ins_cost(BRANCH_COST);
15010 format %{ "cbw$cmp $op1, $labl" %}
15011 ins_encode %{
15012 Label* L = $labl$$label;
15013 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15014 if (cond == Assembler::EQ)
15015 __ cbzw($op1$$Register, *L);
15016 else
15017 __ cbnzw($op1$$Register, *L);
15018 %}
15019 ins_pipe(pipe_cmp_branch);
15020 %}
15021
15022 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15023 match(If cmp (CmpP (DecodeN oop) zero));
15024 effect(USE labl);
15025
15026 ins_cost(BRANCH_COST);
15027 format %{ "cb$cmp $oop, $labl" %}
15028 ins_encode %{
15029 Label* L = $labl$$label;
15030 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15031 if (cond == Assembler::EQ)
15032 __ cbzw($oop$$Register, *L);
15033 else
15034 __ cbnzw($oop$$Register, *L);
15035 %}
15036 ins_pipe(pipe_cmp_branch);
15037 %}
15038
15039 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15040 match(If cmp (CmpU op1 op2));
15041 effect(USE labl);
15042
15043 ins_cost(BRANCH_COST);
15044 format %{ "cbw$cmp $op1, $labl" %}
15045 ins_encode %{
15046 Label* L = $labl$$label;
15047 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15048 if (cond == Assembler::EQ || cond == Assembler::LS) {
15049 __ cbzw($op1$$Register, *L);
15050 } else {
15051 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15052 __ cbnzw($op1$$Register, *L);
15053 }
15054 %}
15055 ins_pipe(pipe_cmp_branch);
15056 %}
15057
15058 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15059 match(If cmp (CmpUL op1 op2));
15060 effect(USE labl);
15061
15062 ins_cost(BRANCH_COST);
15063 format %{ "cb$cmp $op1, $labl" %}
15064 ins_encode %{
15065 Label* L = $labl$$label;
15066 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15067 if (cond == Assembler::EQ || cond == Assembler::LS) {
15068 __ cbz($op1$$Register, *L);
15069 } else {
15070 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15071 __ cbnz($op1$$Register, *L);
15072 }
15073 %}
15074 ins_pipe(pipe_cmp_branch);
15075 %}
15076
15077 // Test bit and Branch
15078
15079 // Patterns for short (< 32KiB) variants
15080 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15081 match(If cmp (CmpL op1 op2));
15082 effect(USE labl);
15083
15084 ins_cost(BRANCH_COST);
15085 format %{ "cb$cmp $op1, $labl # long" %}
15086 ins_encode %{
15087 Label* L = $labl$$label;
15088 Assembler::Condition cond =
15089 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15090 __ tbr(cond, $op1$$Register, 63, *L);
15091 %}
15092 ins_pipe(pipe_cmp_branch);
15093 ins_short_branch(1);
15094 %}
15095
15096 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15097 match(If cmp (CmpI op1 op2));
15098 effect(USE labl);
15099
15100 ins_cost(BRANCH_COST);
15101 format %{ "cb$cmp $op1, $labl # int" %}
15102 ins_encode %{
15103 Label* L = $labl$$label;
15104 Assembler::Condition cond =
15105 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15106 __ tbr(cond, $op1$$Register, 31, *L);
15107 %}
15108 ins_pipe(pipe_cmp_branch);
15109 ins_short_branch(1);
15110 %}
15111
15112 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15113 match(If cmp (CmpL (AndL op1 op2) op3));
15114 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15115 effect(USE labl);
15116
15117 ins_cost(BRANCH_COST);
15118 format %{ "tb$cmp $op1, $op2, $labl" %}
15119 ins_encode %{
15120 Label* L = $labl$$label;
15121 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15122 int bit = exact_log2_long($op2$$constant);
15123 __ tbr(cond, $op1$$Register, bit, *L);
15124 %}
15125 ins_pipe(pipe_cmp_branch);
15126 ins_short_branch(1);
15127 %}
15128
15129 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15130 match(If cmp (CmpI (AndI op1 op2) op3));
15131 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15132 effect(USE labl);
15133
15134 ins_cost(BRANCH_COST);
15135 format %{ "tb$cmp $op1, $op2, $labl" %}
15136 ins_encode %{
15137 Label* L = $labl$$label;
15138 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15139 int bit = exact_log2((juint)$op2$$constant);
15140 __ tbr(cond, $op1$$Register, bit, *L);
15141 %}
15142 ins_pipe(pipe_cmp_branch);
15143 ins_short_branch(1);
15144 %}
15145
15146 // And far variants
15147 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15148 match(If cmp (CmpL op1 op2));
15149 effect(USE labl);
15150
15151 ins_cost(BRANCH_COST);
15152 format %{ "cb$cmp $op1, $labl # long" %}
15153 ins_encode %{
15154 Label* L = $labl$$label;
15155 Assembler::Condition cond =
15156 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15157 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15158 %}
15159 ins_pipe(pipe_cmp_branch);
15160 %}
15161
15162 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15163 match(If cmp (CmpI op1 op2));
15164 effect(USE labl);
15165
15166 ins_cost(BRANCH_COST);
15167 format %{ "cb$cmp $op1, $labl # int" %}
15168 ins_encode %{
15169 Label* L = $labl$$label;
15170 Assembler::Condition cond =
15171 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15172 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15173 %}
15174 ins_pipe(pipe_cmp_branch);
15175 %}
15176
15177 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15178 match(If cmp (CmpL (AndL op1 op2) op3));
15179 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15180 effect(USE labl);
15181
15182 ins_cost(BRANCH_COST);
15183 format %{ "tb$cmp $op1, $op2, $labl" %}
15184 ins_encode %{
15185 Label* L = $labl$$label;
15186 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15187 int bit = exact_log2_long($op2$$constant);
15188 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15189 %}
15190 ins_pipe(pipe_cmp_branch);
15191 %}
15192
15193 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15194 match(If cmp (CmpI (AndI op1 op2) op3));
15195 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15196 effect(USE labl);
15197
15198 ins_cost(BRANCH_COST);
15199 format %{ "tb$cmp $op1, $op2, $labl" %}
15200 ins_encode %{
15201 Label* L = $labl$$label;
15202 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15203 int bit = exact_log2((juint)$op2$$constant);
15204 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15205 %}
15206 ins_pipe(pipe_cmp_branch);
15207 %}
15208
15209 // Test bits
15210
15211 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15212 match(Set cr (CmpL (AndL op1 op2) op3));
15213 predicate(Assembler::operand_valid_for_logical_immediate
15214 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15215
15216 ins_cost(INSN_COST);
15217 format %{ "tst $op1, $op2 # long" %}
15218 ins_encode %{
15219 __ tst($op1$$Register, $op2$$constant);
15220 %}
15221 ins_pipe(ialu_reg_reg);
15222 %}
15223
15224 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15225 match(Set cr (CmpI (AndI op1 op2) op3));
15226 predicate(Assembler::operand_valid_for_logical_immediate
15227 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15228
15229 ins_cost(INSN_COST);
15230 format %{ "tst $op1, $op2 # int" %}
15231 ins_encode %{
15232 __ tstw($op1$$Register, $op2$$constant);
15233 %}
15234 ins_pipe(ialu_reg_reg);
15235 %}
15236
15237 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15238 match(Set cr (CmpL (AndL op1 op2) op3));
15239
15240 ins_cost(INSN_COST);
15241 format %{ "tst $op1, $op2 # long" %}
15242 ins_encode %{
15243 __ tst($op1$$Register, $op2$$Register);
15244 %}
15245 ins_pipe(ialu_reg_reg);
15246 %}
15247
15248 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15249 match(Set cr (CmpI (AndI op1 op2) op3));
15250
15251 ins_cost(INSN_COST);
15252 format %{ "tstw $op1, $op2 # int" %}
15253 ins_encode %{
15254 __ tstw($op1$$Register, $op2$$Register);
15255 %}
15256 ins_pipe(ialu_reg_reg);
15257 %}
15258
15259
15260 // Conditional Far Branch
15261 // Conditional Far Branch Unsigned
15262 // TODO: fixme
15263
15264 // counted loop end branch near
15265 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15266 %{
15267 match(CountedLoopEnd cmp cr);
15268
15269 effect(USE lbl);
15270
15271 ins_cost(BRANCH_COST);
15272 // short variant.
15273 // ins_short_branch(1);
15274 format %{ "b$cmp $lbl \t// counted loop end" %}
15275
15276 ins_encode(aarch64_enc_br_con(cmp, lbl));
15277
15278 ins_pipe(pipe_branch);
15279 %}
15280
15281 // counted loop end branch far
15282 // TODO: fixme
15283
15284 // ============================================================================
15285 // inlined locking and unlocking
15286
15287 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15288 %{
15289 match(Set cr (FastLock object box));
15290 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15291
15292 ins_cost(5 * INSN_COST);
15293 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15294
15295 ins_encode %{
15296 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15297 %}
15298
15299 ins_pipe(pipe_serial);
15300 %}
15301
15302 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15303 %{
15304 match(Set cr (FastUnlock object box));
15305 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15306
15307 ins_cost(5 * INSN_COST);
15308 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15309
15310 ins_encode %{
15311 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15312 %}
15313
15314 ins_pipe(pipe_serial);
15315 %}
15316
15317 // ============================================================================
15318 // Safepoint Instructions
15319
15320 // TODO
15321 // provide a near and far version of this code
15322
15323 instruct safePoint(rFlagsReg cr, iRegP poll)
15324 %{
15325 match(SafePoint poll);
15326 effect(KILL cr);
15327
15328 format %{
15329 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15330 %}
15331 ins_encode %{
15332 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15333 %}
15334 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15335 %}
15336
15337
15338 // ============================================================================
15339 // Procedure Call/Return Instructions
15340
15341 // Call Java Static Instruction
15342
15343 instruct CallStaticJavaDirect(method meth)
15344 %{
15345 match(CallStaticJava);
15346
15347 effect(USE meth);
15348
15349 ins_cost(CALL_COST);
15350
15351 format %{ "call,static $meth \t// ==> " %}
15352
15353 ins_encode(aarch64_enc_java_static_call(meth),
15354 aarch64_enc_call_epilog);
15355
15356 ins_pipe(pipe_class_call);
15357 %}
15358
15359 // TO HERE
15360
15361 // Call Java Dynamic Instruction
15362 instruct CallDynamicJavaDirect(method meth)
15363 %{
15364 match(CallDynamicJava);
15365
15366 effect(USE meth);
15367
15368 ins_cost(CALL_COST);
15369
15370 format %{ "CALL,dynamic $meth \t// ==> " %}
15371
15372 ins_encode(aarch64_enc_java_dynamic_call(meth),
15373 aarch64_enc_call_epilog);
15374
15375 ins_pipe(pipe_class_call);
15376 %}
15377
15378 // Call Runtime Instruction
15379
15380 instruct CallRuntimeDirect(method meth)
15381 %{
15382 match(CallRuntime);
15383
15384 effect(USE meth);
15385
15386 ins_cost(CALL_COST);
15387
15388 format %{ "CALL, runtime $meth" %}
15389
15390 ins_encode( aarch64_enc_java_to_runtime(meth) );
15391
15392 ins_pipe(pipe_class_call);
15393 %}
15394
15395 // Call Runtime Instruction
15396
15397 instruct CallLeafDirect(method meth)
15398 %{
15399 match(CallLeaf);
15400
15401 effect(USE meth);
15402
15403 ins_cost(CALL_COST);
15404
15405 format %{ "CALL, runtime leaf $meth" %}
15406
15407 ins_encode( aarch64_enc_java_to_runtime(meth) );
15408
15409 ins_pipe(pipe_class_call);
15410 %}
15411
15412 // Call Runtime Instruction without safepoint and with vector arguments
15413 instruct CallLeafDirectVector(method meth)
15414 %{
15415 match(CallLeafVector);
15416
15417 effect(USE meth);
15418
15419 ins_cost(CALL_COST);
15420
15421 format %{ "CALL, runtime leaf vector $meth" %}
15422
15423 ins_encode(aarch64_enc_java_to_runtime(meth));
15424
15425 ins_pipe(pipe_class_call);
15426 %}
15427
15428 // Call Runtime Instruction
15429
15430 instruct CallLeafNoFPDirect(method meth)
15431 %{
15432 match(CallLeafNoFP);
15433
15434 effect(USE meth);
15435
15436 ins_cost(CALL_COST);
15437
15438 format %{ "CALL, runtime leaf nofp $meth" %}
15439
15440 ins_encode( aarch64_enc_java_to_runtime(meth) );
15441
15442 ins_pipe(pipe_class_call);
15443 %}
15444
15445 // Tail Call; Jump from runtime stub to Java code.
15446 // Also known as an 'interprocedural jump'.
15447 // Target of jump will eventually return to caller.
15448 // TailJump below removes the return address.
15449 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15450 // emitted just above the TailCall which has reset rfp to the caller state.
15451 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15452 %{
15453 match(TailCall jump_target method_ptr);
15454
15455 ins_cost(CALL_COST);
15456
15457 format %{ "br $jump_target\t# $method_ptr holds method" %}
15458
15459 ins_encode(aarch64_enc_tail_call(jump_target));
15460
15461 ins_pipe(pipe_class_call);
15462 %}
15463
15464 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15465 %{
15466 match(TailJump jump_target ex_oop);
15467
15468 ins_cost(CALL_COST);
15469
15470 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15471
15472 ins_encode(aarch64_enc_tail_jmp(jump_target));
15473
15474 ins_pipe(pipe_class_call);
15475 %}
15476
15477 // Forward exception.
15478 instruct ForwardExceptionjmp()
15479 %{
15480 match(ForwardException);
15481 ins_cost(CALL_COST);
15482
15483 format %{ "b forward_exception_stub" %}
15484 ins_encode %{
15485 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15486 %}
15487 ins_pipe(pipe_class_call);
15488 %}
15489
15490 // Create exception oop: created by stack-crawling runtime code.
15491 // Created exception is now available to this handler, and is setup
15492 // just prior to jumping to this handler. No code emitted.
15493 // TODO check
15494 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15495 instruct CreateException(iRegP_R0 ex_oop)
15496 %{
15497 match(Set ex_oop (CreateEx));
15498
15499 format %{ " -- \t// exception oop; no code emitted" %}
15500
15501 size(0);
15502
15503 ins_encode( /*empty*/ );
15504
15505 ins_pipe(pipe_class_empty);
15506 %}
15507
15508 // Rethrow exception: The exception oop will come in the first
15509 // argument position. Then JUMP (not call) to the rethrow stub code.
15510 instruct RethrowException() %{
15511 match(Rethrow);
15512 ins_cost(CALL_COST);
15513
15514 format %{ "b rethrow_stub" %}
15515
15516 ins_encode( aarch64_enc_rethrow() );
15517
15518 ins_pipe(pipe_class_call);
15519 %}
15520
15521
15522 // Return Instruction
15523 // epilog node loads ret address into lr as part of frame pop
15524 instruct Ret()
15525 %{
15526 match(Return);
15527
15528 format %{ "ret\t// return register" %}
15529
15530 ins_encode( aarch64_enc_ret() );
15531
15532 ins_pipe(pipe_branch);
15533 %}
15534
15535 // Die now.
15536 instruct ShouldNotReachHere() %{
15537 match(Halt);
15538
15539 ins_cost(CALL_COST);
15540 format %{ "ShouldNotReachHere" %}
15541
15542 ins_encode %{
15543 if (is_reachable()) {
15544 const char* str = __ code_string(_halt_reason);
15545 __ stop(str);
15546 }
15547 %}
15548
15549 ins_pipe(pipe_class_default);
15550 %}
15551
15552 // ============================================================================
15553 // Partial Subtype Check
15554 //
15555 // superklass array for an instance of the superklass. Set a hidden
15556 // internal cache on a hit (cache is checked with exposed code in
15557 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15558 // encoding ALSO sets flags.
15559
15560 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15561 %{
15562 match(Set result (PartialSubtypeCheck sub super));
15563 predicate(!UseSecondarySupersTable);
15564 effect(KILL cr, KILL temp);
15565
15566 ins_cost(20 * INSN_COST); // slightly larger than the next version
15567 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15568
15569 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15570
15571 opcode(0x1); // Force zero of result reg on hit
15572
15573 ins_pipe(pipe_class_memory);
15574 %}
15575
15576 // Two versions of partialSubtypeCheck, both used when we need to
15577 // search for a super class in the secondary supers array. The first
15578 // is used when we don't know _a priori_ the class being searched
15579 // for. The second, far more common, is used when we do know: this is
15580 // used for instanceof, checkcast, and any case where C2 can determine
15581 // it by constant propagation.
15582
15583 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15584 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15585 rFlagsReg cr)
15586 %{
15587 match(Set result (PartialSubtypeCheck sub super));
15588 predicate(UseSecondarySupersTable);
15589 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15590
15591 ins_cost(10 * INSN_COST); // slightly larger than the next version
15592 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15593
15594 ins_encode %{
15595 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15596 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15597 $vtemp$$FloatRegister,
15598 $result$$Register, /*L_success*/nullptr);
15599 %}
15600
15601 ins_pipe(pipe_class_memory);
15602 %}
15603
15604 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15605 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15606 rFlagsReg cr)
15607 %{
15608 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15609 predicate(UseSecondarySupersTable);
15610 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15611
15612 ins_cost(5 * INSN_COST); // smaller than the next version
15613 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15614
15615 ins_encode %{
15616 bool success = false;
15617 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15618 if (InlineSecondarySupersTest) {
15619 success =
15620 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15621 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15622 $vtemp$$FloatRegister,
15623 $result$$Register,
15624 super_klass_slot);
15625 } else {
15626 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15627 success = (call != nullptr);
15628 }
15629 if (!success) {
15630 ciEnv::current()->record_failure("CodeCache is full");
15631 return;
15632 }
15633 %}
15634
15635 ins_pipe(pipe_class_memory);
15636 %}
15637
15638 // Intrisics for String.compareTo()
15639
15640 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15641 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15642 %{
15643 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15644 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15645 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15646
15647 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15648 ins_encode %{
15649 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15650 __ string_compare($str1$$Register, $str2$$Register,
15651 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15652 $tmp1$$Register, $tmp2$$Register,
15653 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15654 %}
15655 ins_pipe(pipe_class_memory);
15656 %}
15657
15658 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15659 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15660 %{
15661 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15662 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15663 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15664
15665 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15666 ins_encode %{
15667 __ string_compare($str1$$Register, $str2$$Register,
15668 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15669 $tmp1$$Register, $tmp2$$Register,
15670 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15671 %}
15672 ins_pipe(pipe_class_memory);
15673 %}
15674
15675 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15676 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15677 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15678 %{
15679 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15680 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15681 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15682 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15683
15684 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15685 ins_encode %{
15686 __ string_compare($str1$$Register, $str2$$Register,
15687 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15688 $tmp1$$Register, $tmp2$$Register,
15689 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15690 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15691 %}
15692 ins_pipe(pipe_class_memory);
15693 %}
15694
15695 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15696 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15697 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15698 %{
15699 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15700 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15701 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15702 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15703
15704 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15705 ins_encode %{
15706 __ string_compare($str1$$Register, $str2$$Register,
15707 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15708 $tmp1$$Register, $tmp2$$Register,
15709 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15710 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15711 %}
15712 ins_pipe(pipe_class_memory);
15713 %}
15714
15715 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15716 // these string_compare variants as NEON register type for convenience so that the prototype of
15717 // string_compare can be shared with all variants.
15718
15719 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15720 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15721 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15722 pRegGov_P1 pgtmp2, rFlagsReg cr)
15723 %{
15724 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15725 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15726 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15727 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15728
15729 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15730 ins_encode %{
15731 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15732 __ string_compare($str1$$Register, $str2$$Register,
15733 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15734 $tmp1$$Register, $tmp2$$Register,
15735 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15736 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15737 StrIntrinsicNode::LL);
15738 %}
15739 ins_pipe(pipe_class_memory);
15740 %}
15741
15742 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15743 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15744 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15745 pRegGov_P1 pgtmp2, rFlagsReg cr)
15746 %{
15747 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15748 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15749 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15750 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15751
15752 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15753 ins_encode %{
15754 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15755 __ string_compare($str1$$Register, $str2$$Register,
15756 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15757 $tmp1$$Register, $tmp2$$Register,
15758 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15759 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15760 StrIntrinsicNode::LU);
15761 %}
15762 ins_pipe(pipe_class_memory);
15763 %}
15764
15765 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15766 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15767 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15768 pRegGov_P1 pgtmp2, rFlagsReg cr)
15769 %{
15770 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15771 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15772 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15773 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15774
15775 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15776 ins_encode %{
15777 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15778 __ string_compare($str1$$Register, $str2$$Register,
15779 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15780 $tmp1$$Register, $tmp2$$Register,
15781 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15782 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15783 StrIntrinsicNode::UL);
15784 %}
15785 ins_pipe(pipe_class_memory);
15786 %}
15787
15788 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15789 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15790 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15791 pRegGov_P1 pgtmp2, rFlagsReg cr)
15792 %{
15793 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15794 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15795 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15796 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15797
15798 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15799 ins_encode %{
15800 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15801 __ string_compare($str1$$Register, $str2$$Register,
15802 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15803 $tmp1$$Register, $tmp2$$Register,
15804 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15805 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15806 StrIntrinsicNode::UU);
15807 %}
15808 ins_pipe(pipe_class_memory);
15809 %}
15810
15811 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15812 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15813 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15814 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15815 %{
15816 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15817 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15818 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15819 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15820 TEMP vtmp0, TEMP vtmp1, KILL cr);
15821 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15822 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15823
15824 ins_encode %{
15825 __ string_indexof($str1$$Register, $str2$$Register,
15826 $cnt1$$Register, $cnt2$$Register,
15827 $tmp1$$Register, $tmp2$$Register,
15828 $tmp3$$Register, $tmp4$$Register,
15829 $tmp5$$Register, $tmp6$$Register,
15830 -1, $result$$Register, StrIntrinsicNode::UU);
15831 %}
15832 ins_pipe(pipe_class_memory);
15833 %}
15834
15835 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15836 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15837 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15838 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15839 %{
15840 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15841 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15842 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15843 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15844 TEMP vtmp0, TEMP vtmp1, KILL cr);
15845 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15846 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15847
15848 ins_encode %{
15849 __ string_indexof($str1$$Register, $str2$$Register,
15850 $cnt1$$Register, $cnt2$$Register,
15851 $tmp1$$Register, $tmp2$$Register,
15852 $tmp3$$Register, $tmp4$$Register,
15853 $tmp5$$Register, $tmp6$$Register,
15854 -1, $result$$Register, StrIntrinsicNode::LL);
15855 %}
15856 ins_pipe(pipe_class_memory);
15857 %}
15858
15859 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15860 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15861 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15862 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15863 %{
15864 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15865 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15866 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15867 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15868 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15869 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15870 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15871
15872 ins_encode %{
15873 __ string_indexof($str1$$Register, $str2$$Register,
15874 $cnt1$$Register, $cnt2$$Register,
15875 $tmp1$$Register, $tmp2$$Register,
15876 $tmp3$$Register, $tmp4$$Register,
15877 $tmp5$$Register, $tmp6$$Register,
15878 -1, $result$$Register, StrIntrinsicNode::UL);
15879 %}
15880 ins_pipe(pipe_class_memory);
15881 %}
15882
15883 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15884 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15885 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15886 %{
15887 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15888 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15889 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15890 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15891 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15892 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15893
15894 ins_encode %{
15895 int icnt2 = (int)$int_cnt2$$constant;
15896 __ string_indexof($str1$$Register, $str2$$Register,
15897 $cnt1$$Register, zr,
15898 $tmp1$$Register, $tmp2$$Register,
15899 $tmp3$$Register, $tmp4$$Register, zr, zr,
15900 icnt2, $result$$Register, StrIntrinsicNode::UU);
15901 %}
15902 ins_pipe(pipe_class_memory);
15903 %}
15904
15905 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15906 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15907 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15908 %{
15909 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15910 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15911 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15912 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15913 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15914 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15915
15916 ins_encode %{
15917 int icnt2 = (int)$int_cnt2$$constant;
15918 __ string_indexof($str1$$Register, $str2$$Register,
15919 $cnt1$$Register, zr,
15920 $tmp1$$Register, $tmp2$$Register,
15921 $tmp3$$Register, $tmp4$$Register, zr, zr,
15922 icnt2, $result$$Register, StrIntrinsicNode::LL);
15923 %}
15924 ins_pipe(pipe_class_memory);
15925 %}
15926
15927 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15928 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15929 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15930 %{
15931 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15932 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15933 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15934 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15935 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15936 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15937
15938 ins_encode %{
15939 int icnt2 = (int)$int_cnt2$$constant;
15940 __ string_indexof($str1$$Register, $str2$$Register,
15941 $cnt1$$Register, zr,
15942 $tmp1$$Register, $tmp2$$Register,
15943 $tmp3$$Register, $tmp4$$Register, zr, zr,
15944 icnt2, $result$$Register, StrIntrinsicNode::UL);
15945 %}
15946 ins_pipe(pipe_class_memory);
15947 %}
15948
15949 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15950 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15951 iRegINoSp tmp3, rFlagsReg cr)
15952 %{
15953 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15954 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15955 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15956 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15957
15958 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15959
15960 ins_encode %{
15961 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15962 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15963 $tmp3$$Register);
15964 %}
15965 ins_pipe(pipe_class_memory);
15966 %}
15967
15968 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15969 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15970 iRegINoSp tmp3, rFlagsReg cr)
15971 %{
15972 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15973 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15974 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15975 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15976
15977 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15978
15979 ins_encode %{
15980 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15981 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15982 $tmp3$$Register);
15983 %}
15984 ins_pipe(pipe_class_memory);
15985 %}
15986
15987 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15988 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15989 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15990 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15991 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15992 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15993 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15994 ins_encode %{
15995 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15996 $result$$Register, $ztmp1$$FloatRegister,
15997 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15998 $ptmp$$PRegister, true /* isL */);
15999 %}
16000 ins_pipe(pipe_class_memory);
16001 %}
16002
16003 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16004 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16005 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16006 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16007 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16008 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16009 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16010 ins_encode %{
16011 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16012 $result$$Register, $ztmp1$$FloatRegister,
16013 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16014 $ptmp$$PRegister, false /* isL */);
16015 %}
16016 ins_pipe(pipe_class_memory);
16017 %}
16018
16019 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16020 iRegI_R0 result, rFlagsReg cr)
16021 %{
16022 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16023 match(Set result (StrEquals (Binary str1 str2) cnt));
16024 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16025
16026 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16027 ins_encode %{
16028 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16029 __ string_equals($str1$$Register, $str2$$Register,
16030 $result$$Register, $cnt$$Register);
16031 %}
16032 ins_pipe(pipe_class_memory);
16033 %}
16034
16035 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16036 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16037 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16038 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16039 iRegP_R10 tmp, rFlagsReg cr)
16040 %{
16041 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16042 match(Set result (AryEq ary1 ary2));
16043 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16044 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16045 TEMP vtmp6, TEMP vtmp7, KILL cr);
16046
16047 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16048 ins_encode %{
16049 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16050 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16051 $result$$Register, $tmp$$Register, 1);
16052 if (tpc == nullptr) {
16053 ciEnv::current()->record_failure("CodeCache is full");
16054 return;
16055 }
16056 %}
16057 ins_pipe(pipe_class_memory);
16058 %}
16059
16060 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16061 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16062 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16063 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16064 iRegP_R10 tmp, rFlagsReg cr)
16065 %{
16066 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16067 match(Set result (AryEq ary1 ary2));
16068 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16069 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16070 TEMP vtmp6, TEMP vtmp7, KILL cr);
16071
16072 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16073 ins_encode %{
16074 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16075 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16076 $result$$Register, $tmp$$Register, 2);
16077 if (tpc == nullptr) {
16078 ciEnv::current()->record_failure("CodeCache is full");
16079 return;
16080 }
16081 %}
16082 ins_pipe(pipe_class_memory);
16083 %}
16084
16085 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16086 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16087 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16088 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16089 %{
16090 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16091 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16092 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16093
16094 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16095 ins_encode %{
16096 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16097 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16098 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16099 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16100 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16101 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16102 (BasicType)$basic_type$$constant);
16103 if (tpc == nullptr) {
16104 ciEnv::current()->record_failure("CodeCache is full");
16105 return;
16106 }
16107 %}
16108 ins_pipe(pipe_class_memory);
16109 %}
16110
16111 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16112 %{
16113 match(Set result (CountPositives ary1 len));
16114 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16115 format %{ "count positives byte[] $ary1,$len -> $result" %}
16116 ins_encode %{
16117 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16118 if (tpc == nullptr) {
16119 ciEnv::current()->record_failure("CodeCache is full");
16120 return;
16121 }
16122 %}
16123 ins_pipe( pipe_slow );
16124 %}
16125
16126 // fast char[] to byte[] compression
16127 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16128 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16129 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16130 iRegI_R0 result, rFlagsReg cr)
16131 %{
16132 match(Set result (StrCompressedCopy src (Binary dst len)));
16133 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16134 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16135
16136 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16137 ins_encode %{
16138 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16139 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16140 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16141 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16142 %}
16143 ins_pipe(pipe_slow);
16144 %}
16145
16146 // fast byte[] to char[] inflation
16147 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16148 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16149 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16150 %{
16151 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16152 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16153 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16154 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16155
16156 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16157 ins_encode %{
16158 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16159 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16160 $vtmp2$$FloatRegister, $tmp$$Register);
16161 if (tpc == nullptr) {
16162 ciEnv::current()->record_failure("CodeCache is full");
16163 return;
16164 }
16165 %}
16166 ins_pipe(pipe_class_memory);
16167 %}
16168
16169 // encode char[] to byte[] in ISO_8859_1
16170 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16171 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16172 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16173 iRegI_R0 result, rFlagsReg cr)
16174 %{
16175 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16176 match(Set result (EncodeISOArray src (Binary dst len)));
16177 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16178 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16179
16180 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16181 ins_encode %{
16182 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16183 $result$$Register, false,
16184 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16185 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16186 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16187 %}
16188 ins_pipe(pipe_class_memory);
16189 %}
16190
16191 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16192 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16193 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16194 iRegI_R0 result, rFlagsReg cr)
16195 %{
16196 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16197 match(Set result (EncodeISOArray src (Binary dst len)));
16198 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16199 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16200
16201 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16202 ins_encode %{
16203 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16204 $result$$Register, true,
16205 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16206 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16207 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16208 %}
16209 ins_pipe(pipe_class_memory);
16210 %}
16211
16212 //----------------------------- CompressBits/ExpandBits ------------------------
16213
16214 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16215 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16216 match(Set dst (CompressBits src mask));
16217 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16218 format %{ "mov $tsrc, $src\n\t"
16219 "mov $tmask, $mask\n\t"
16220 "bext $tdst, $tsrc, $tmask\n\t"
16221 "mov $dst, $tdst"
16222 %}
16223 ins_encode %{
16224 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16225 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16226 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16227 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16228 %}
16229 ins_pipe(pipe_slow);
16230 %}
16231
16232 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16233 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16234 match(Set dst (CompressBits (LoadI mem) mask));
16235 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16236 format %{ "ldrs $tsrc, $mem\n\t"
16237 "ldrs $tmask, $mask\n\t"
16238 "bext $tdst, $tsrc, $tmask\n\t"
16239 "mov $dst, $tdst"
16240 %}
16241 ins_encode %{
16242 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16243 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16244 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16245 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16246 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16247 %}
16248 ins_pipe(pipe_slow);
16249 %}
16250
16251 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16252 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16253 match(Set dst (CompressBits src mask));
16254 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16255 format %{ "mov $tsrc, $src\n\t"
16256 "mov $tmask, $mask\n\t"
16257 "bext $tdst, $tsrc, $tmask\n\t"
16258 "mov $dst, $tdst"
16259 %}
16260 ins_encode %{
16261 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16262 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16263 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16264 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16265 %}
16266 ins_pipe(pipe_slow);
16267 %}
16268
16269 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16270 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16271 match(Set dst (CompressBits (LoadL mem) mask));
16272 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16273 format %{ "ldrd $tsrc, $mem\n\t"
16274 "ldrd $tmask, $mask\n\t"
16275 "bext $tdst, $tsrc, $tmask\n\t"
16276 "mov $dst, $tdst"
16277 %}
16278 ins_encode %{
16279 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16280 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16281 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16282 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16283 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16284 %}
16285 ins_pipe(pipe_slow);
16286 %}
16287
16288 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16289 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16290 match(Set dst (ExpandBits src mask));
16291 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16292 format %{ "mov $tsrc, $src\n\t"
16293 "mov $tmask, $mask\n\t"
16294 "bdep $tdst, $tsrc, $tmask\n\t"
16295 "mov $dst, $tdst"
16296 %}
16297 ins_encode %{
16298 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16299 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16300 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16301 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16302 %}
16303 ins_pipe(pipe_slow);
16304 %}
16305
16306 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16307 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16308 match(Set dst (ExpandBits (LoadI mem) mask));
16309 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16310 format %{ "ldrs $tsrc, $mem\n\t"
16311 "ldrs $tmask, $mask\n\t"
16312 "bdep $tdst, $tsrc, $tmask\n\t"
16313 "mov $dst, $tdst"
16314 %}
16315 ins_encode %{
16316 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16317 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16318 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16319 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16320 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16321 %}
16322 ins_pipe(pipe_slow);
16323 %}
16324
16325 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16326 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16327 match(Set dst (ExpandBits src mask));
16328 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16329 format %{ "mov $tsrc, $src\n\t"
16330 "mov $tmask, $mask\n\t"
16331 "bdep $tdst, $tsrc, $tmask\n\t"
16332 "mov $dst, $tdst"
16333 %}
16334 ins_encode %{
16335 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16336 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16337 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16338 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16339 %}
16340 ins_pipe(pipe_slow);
16341 %}
16342
16343
16344 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16345 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16346 match(Set dst (ExpandBits (LoadL mem) mask));
16347 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16348 format %{ "ldrd $tsrc, $mem\n\t"
16349 "ldrd $tmask, $mask\n\t"
16350 "bdep $tdst, $tsrc, $tmask\n\t"
16351 "mov $dst, $tdst"
16352 %}
16353 ins_encode %{
16354 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16355 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16356 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16357 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16358 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16359 %}
16360 ins_pipe(pipe_slow);
16361 %}
16362
16363 //----------------------------- Reinterpret ----------------------------------
16364 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16365 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16366 match(Set dst (ReinterpretHF2S src));
16367 format %{ "reinterpretHF2S $dst, $src" %}
16368 ins_encode %{
16369 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16370 %}
16371 ins_pipe(pipe_slow);
16372 %}
16373
16374 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16375 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16376 match(Set dst (ReinterpretS2HF src));
16377 format %{ "reinterpretS2HF $dst, $src" %}
16378 ins_encode %{
16379 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16380 %}
16381 ins_pipe(pipe_slow);
16382 %}
16383
16384 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16385 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16386 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16387 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16388 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16389 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16390 // can be omitted in this pattern, resulting in -
16391 // fcvt $dst, $src // Convert float to half-precision float
16392 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16393 %{
16394 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16395 format %{ "convF2HFAndS2HF $dst, $src" %}
16396 ins_encode %{
16397 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16398 %}
16399 ins_pipe(pipe_slow);
16400 %}
16401
16402 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16403 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16404 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16405 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16406 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16407 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16408 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16409 // resulting in -
16410 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16411 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16412 %{
16413 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16414 format %{ "convHF2SAndHF2F $dst, $src" %}
16415 ins_encode %{
16416 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16417 %}
16418 ins_pipe(pipe_slow);
16419 %}
16420
16421 // ============================================================================
16422 // This name is KNOWN by the ADLC and cannot be changed.
16423 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16424 // for this guy.
16425 instruct tlsLoadP(thread_RegP dst)
16426 %{
16427 match(Set dst (ThreadLocal));
16428
16429 ins_cost(0);
16430
16431 format %{ " -- \t// $dst=Thread::current(), empty" %}
16432
16433 size(0);
16434
16435 ins_encode( /*empty*/ );
16436
16437 ins_pipe(pipe_class_empty);
16438 %}
16439
16440 //----------PEEPHOLE RULES-----------------------------------------------------
16441 // These must follow all instruction definitions as they use the names
16442 // defined in the instructions definitions.
16443 //
16444 // peepmatch ( root_instr_name [preceding_instruction]* );
16445 //
16446 // peepconstraint %{
16447 // (instruction_number.operand_name relational_op instruction_number.operand_name
16448 // [, ...] );
16449 // // instruction numbers are zero-based using left to right order in peepmatch
16450 //
16451 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16452 // // provide an instruction_number.operand_name for each operand that appears
16453 // // in the replacement instruction's match rule
16454 //
16455 // ---------VM FLAGS---------------------------------------------------------
16456 //
16457 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16458 //
16459 // Each peephole rule is given an identifying number starting with zero and
16460 // increasing by one in the order seen by the parser. An individual peephole
16461 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16462 // on the command-line.
16463 //
16464 // ---------CURRENT LIMITATIONS----------------------------------------------
16465 //
16466 // Only match adjacent instructions in same basic block
16467 // Only equality constraints
16468 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16469 // Only one replacement instruction
16470 //
16471 // ---------EXAMPLE----------------------------------------------------------
16472 //
16473 // // pertinent parts of existing instructions in architecture description
16474 // instruct movI(iRegINoSp dst, iRegI src)
16475 // %{
16476 // match(Set dst (CopyI src));
16477 // %}
16478 //
16479 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16480 // %{
16481 // match(Set dst (AddI dst src));
16482 // effect(KILL cr);
16483 // %}
16484 //
16485 // // Change (inc mov) to lea
16486 // peephole %{
16487 // // increment preceded by register-register move
16488 // peepmatch ( incI_iReg movI );
16489 // // require that the destination register of the increment
16490 // // match the destination register of the move
16491 // peepconstraint ( 0.dst == 1.dst );
16492 // // construct a replacement instruction that sets
16493 // // the destination to ( move's source register + one )
16494 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16495 // %}
16496 //
16497
16498 // Implementation no longer uses movX instructions since
16499 // machine-independent system no longer uses CopyX nodes.
16500 //
16501 // peephole
16502 // %{
16503 // peepmatch (incI_iReg movI);
16504 // peepconstraint (0.dst == 1.dst);
16505 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16506 // %}
16507
16508 // peephole
16509 // %{
16510 // peepmatch (decI_iReg movI);
16511 // peepconstraint (0.dst == 1.dst);
16512 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16513 // %}
16514
16515 // peephole
16516 // %{
16517 // peepmatch (addI_iReg_imm movI);
16518 // peepconstraint (0.dst == 1.dst);
16519 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16520 // %}
16521
16522 // peephole
16523 // %{
16524 // peepmatch (incL_iReg movL);
16525 // peepconstraint (0.dst == 1.dst);
16526 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16527 // %}
16528
16529 // peephole
16530 // %{
16531 // peepmatch (decL_iReg movL);
16532 // peepconstraint (0.dst == 1.dst);
16533 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16534 // %}
16535
16536 // peephole
16537 // %{
16538 // peepmatch (addL_iReg_imm movL);
16539 // peepconstraint (0.dst == 1.dst);
16540 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16541 // %}
16542
16543 // peephole
16544 // %{
16545 // peepmatch (addP_iReg_imm movP);
16546 // peepconstraint (0.dst == 1.dst);
16547 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16548 // %}
16549
16550 // // Change load of spilled value to only a spill
16551 // instruct storeI(memory mem, iRegI src)
16552 // %{
16553 // match(Set mem (StoreI mem src));
16554 // %}
16555 //
16556 // instruct loadI(iRegINoSp dst, memory mem)
16557 // %{
16558 // match(Set dst (LoadI mem));
16559 // %}
16560 //
16561
16562 //----------SMARTSPILL RULES---------------------------------------------------
16563 // These must follow all instruction definitions as they use the names
16564 // defined in the instructions definitions.
16565
16566 // Local Variables:
16567 // mode: c++
16568 // End: