1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_CompareAndSwapB:
1512 case Op_CompareAndSwapS:
1513 case Op_GetAndSetI:
1514 case Op_GetAndSetL:
1515 case Op_GetAndSetP:
1516 case Op_GetAndSetN:
1517 case Op_GetAndAddI:
1518 case Op_GetAndAddL:
1519 return true;
1520 case Op_CompareAndExchangeI:
1521 case Op_CompareAndExchangeN:
1522 case Op_CompareAndExchangeB:
1523 case Op_CompareAndExchangeS:
1524 case Op_CompareAndExchangeL:
1525 case Op_CompareAndExchangeP:
1526 case Op_WeakCompareAndSwapB:
1527 case Op_WeakCompareAndSwapS:
1528 case Op_WeakCompareAndSwapI:
1529 case Op_WeakCompareAndSwapL:
1530 case Op_WeakCompareAndSwapP:
1531 case Op_WeakCompareAndSwapN:
1532 return maybe_volatile;
1533 default:
1534 return false;
1535 }
1536 }
1537
1538 // helper to determine the maximum number of Phi nodes we may need to
1539 // traverse when searching from a card mark membar for the merge mem
1540 // feeding a trailing membar or vice versa
1541
1542 // predicates controlling emit of ldr<x>/ldar<x>
1543
1544 bool unnecessary_acquire(const Node *barrier)
1545 {
1546 assert(barrier->is_MemBar(), "expecting a membar");
1547
1548 MemBarNode* mb = barrier->as_MemBar();
1549
1550 if (mb->trailing_load()) {
1551 return true;
1552 }
1553
1554 if (mb->trailing_load_store()) {
1555 Node* load_store = mb->in(MemBarNode::Precedent);
1556 assert(load_store->is_LoadStore(), "unexpected graph shape");
1557 return is_CAS(load_store->Opcode(), true);
1558 }
1559
1560 return false;
1561 }
1562
1563 bool needs_acquiring_load(const Node *n)
1564 {
1565 assert(n->is_Load(), "expecting a load");
1566 LoadNode *ld = n->as_Load();
1567 return ld->is_acquire();
1568 }
1569
1570 bool unnecessary_release(const Node *n)
1571 {
1572 assert((n->is_MemBar() &&
1573 n->Opcode() == Op_MemBarRelease),
1574 "expecting a release membar");
1575
1576 MemBarNode *barrier = n->as_MemBar();
1577 if (!barrier->leading()) {
1578 return false;
1579 } else {
1580 Node* trailing = barrier->trailing_membar();
1581 MemBarNode* trailing_mb = trailing->as_MemBar();
1582 assert(trailing_mb->trailing(), "Not a trailing membar?");
1583 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1584
1585 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1586 if (mem->is_Store()) {
1587 assert(mem->as_Store()->is_release(), "");
1588 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1589 return true;
1590 } else {
1591 assert(mem->is_LoadStore(), "");
1592 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1593 return is_CAS(mem->Opcode(), true);
1594 }
1595 }
1596 return false;
1597 }
1598
1599 bool unnecessary_volatile(const Node *n)
1600 {
1601 // assert n->is_MemBar();
1602 MemBarNode *mbvol = n->as_MemBar();
1603
1604 bool release = mbvol->trailing_store();
1605 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1606 #ifdef ASSERT
1607 if (release) {
1608 Node* leading = mbvol->leading_membar();
1609 assert(leading->Opcode() == Op_MemBarRelease, "");
1610 assert(leading->as_MemBar()->leading_store(), "");
1611 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1612 }
1613 #endif
1614
1615 return release;
1616 }
1617
1618 // predicates controlling emit of str<x>/stlr<x>
1619
1620 bool needs_releasing_store(const Node *n)
1621 {
1622 // assert n->is_Store();
1623 StoreNode *st = n->as_Store();
1624 return st->trailing_membar() != nullptr;
1625 }
1626
1627 // predicate controlling translation of CAS
1628 //
1629 // returns true if CAS needs to use an acquiring load otherwise false
1630
1631 bool needs_acquiring_load_exclusive(const Node *n)
1632 {
1633 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1634 LoadStoreNode* ldst = n->as_LoadStore();
1635 if (is_CAS(n->Opcode(), false)) {
1636 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1637 } else {
1638 return ldst->trailing_membar() != nullptr;
1639 }
1640
1641 // so we can just return true here
1642 return true;
1643 }
1644
1645 #define __ masm->
1646
1647 // advance declarations for helper functions to convert register
1648 // indices to register objects
1649
1650 // the ad file has to provide implementations of certain methods
1651 // expected by the generic code
1652 //
1653 // REQUIRED FUNCTIONALITY
1654
1655 //=============================================================================
1656
1657 // !!!!! Special hack to get all types of calls to specify the byte offset
1658 // from the start of the call to the point where the return address
1659 // will point.
1660
1661 int MachCallStaticJavaNode::ret_addr_offset()
1662 {
1663 // call should be a simple bl
1664 int off = 4;
1665 return off;
1666 }
1667
1668 int MachCallDynamicJavaNode::ret_addr_offset()
1669 {
1670 return 16; // movz, movk, movk, bl
1671 }
1672
1673 int MachCallRuntimeNode::ret_addr_offset() {
1674 // for generated stubs the call will be
1675 // bl(addr)
1676 // or with far branches
1677 // bl(trampoline_stub)
1678 // for real runtime callouts it will be six instructions
1679 // see aarch64_enc_java_to_runtime
1680 // adr(rscratch2, retaddr)
1681 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1682 // lea(rscratch1, RuntimeAddress(addr)
1683 // blr(rscratch1)
1684 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1685 if (cb) {
1686 return 1 * NativeInstruction::instruction_size;
1687 } else if (_entry_point == nullptr) {
1688 // See CallLeafNoFPIndirect
1689 return 1 * NativeInstruction::instruction_size;
1690 } else {
1691 return 6 * NativeInstruction::instruction_size;
1692 }
1693 }
1694
1695 //=============================================================================
1696
1697 #ifndef PRODUCT
1698 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1699 st->print("BREAKPOINT");
1700 }
1701 #endif
1702
1703 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1704 __ brk(0);
1705 }
1706
1707 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1708 return MachNode::size(ra_);
1709 }
1710
1711 //=============================================================================
1712
1713 #ifndef PRODUCT
1714 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1715 st->print("nop \t# %d bytes pad for loops and calls", _count);
1716 }
1717 #endif
1718
1719 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1720 for (int i = 0; i < _count; i++) {
1721 __ nop();
1722 }
1723 }
1724
1725 uint MachNopNode::size(PhaseRegAlloc*) const {
1726 return _count * NativeInstruction::instruction_size;
1727 }
1728
1729 //=============================================================================
1730 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1731
1732 int ConstantTable::calculate_table_base_offset() const {
1733 return 0; // absolute addressing, no offset
1734 }
1735
1736 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1737 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1738 ShouldNotReachHere();
1739 }
1740
1741 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1742 // Empty encoding
1743 }
1744
1745 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1746 return 0;
1747 }
1748
1749 #ifndef PRODUCT
1750 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1751 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1752 }
1753 #endif
1754
1755 #ifndef PRODUCT
1756 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1757 Compile* C = ra_->C;
1758
1759 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1760
1761 if (C->output()->need_stack_bang(framesize))
1762 st->print("# stack bang size=%d\n\t", framesize);
1763
1764 if (VM_Version::use_rop_protection()) {
1765 st->print("ldr zr, [lr]\n\t");
1766 st->print("paciaz\n\t");
1767 }
1768 if (framesize < ((1 << 9) + 2 * wordSize)) {
1769 st->print("sub sp, sp, #%d\n\t", framesize);
1770 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1771 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1772 } else {
1773 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1774 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1775 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1776 st->print("sub sp, sp, rscratch1");
1777 }
1778 if (C->stub_function() == nullptr) {
1779 st->print("\n\t");
1780 st->print("ldr rscratch1, [guard]\n\t");
1781 st->print("dmb ishld\n\t");
1782 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1783 st->print("cmp rscratch1, rscratch2\n\t");
1784 st->print("b.eq skip");
1785 st->print("\n\t");
1786 st->print("blr #nmethod_entry_barrier_stub\n\t");
1787 st->print("b skip\n\t");
1788 st->print("guard: int\n\t");
1789 st->print("\n\t");
1790 st->print("skip:\n\t");
1791 }
1792 }
1793 #endif
1794
1795 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1796 Compile* C = ra_->C;
1797
1798
1799 __ verified_entry(C, 0);
1800
1801 if (C->stub_function() == nullptr) {
1802 __ entry_barrier();
1803 }
1804
1805 if (!Compile::current()->output()->in_scratch_emit_size()) {
1806 __ bind(*_verified_entry);
1807 }
1808
1809 if (VerifyStackAtCalls) {
1810 Unimplemented();
1811 }
1812
1813 C->output()->set_frame_complete(__ offset());
1814
1815 if (C->has_mach_constant_base_node()) {
1816 // NOTE: We set the table base offset here because users might be
1817 // emitted before MachConstantBaseNode.
1818 ConstantTable& constant_table = C->output()->constant_table();
1819 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1820 }
1821 }
1822
1823 int MachPrologNode::reloc() const
1824 {
1825 return 0;
1826 }
1827
1828 //=============================================================================
1829
1830 #ifndef PRODUCT
1831 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1832 Compile* C = ra_->C;
1833 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1834
1835 st->print("# pop frame %d\n\t",framesize);
1836
1837 if (framesize == 0) {
1838 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1839 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1840 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1841 st->print("add sp, sp, #%d\n\t", framesize);
1842 } else {
1843 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1844 st->print("add sp, sp, rscratch1\n\t");
1845 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1846 }
1847 if (VM_Version::use_rop_protection()) {
1848 st->print("autiaz\n\t");
1849 st->print("ldr zr, [lr]\n\t");
1850 }
1851
1852 if (do_polling() && C->is_method_compilation()) {
1853 st->print("# test polling word\n\t");
1854 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1855 st->print("cmp sp, rscratch1\n\t");
1856 st->print("bhi #slow_path");
1857 }
1858 }
1859 #endif
1860
1861 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1862 Compile* C = ra_->C;
1863 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1864
1865 __ remove_frame(framesize, C->needs_stack_repair());
1866
1867 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1868 __ reserved_stack_check();
1869 }
1870
1871 if (do_polling() && C->is_method_compilation()) {
1872 Label dummy_label;
1873 Label* code_stub = &dummy_label;
1874 if (!C->output()->in_scratch_emit_size()) {
1875 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1876 C->output()->add_stub(stub);
1877 code_stub = &stub->entry();
1878 }
1879 __ relocate(relocInfo::poll_return_type);
1880 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1881 }
1882 }
1883
1884 int MachEpilogNode::reloc() const {
1885 // Return number of relocatable values contained in this instruction.
1886 return 1; // 1 for polling page.
1887 }
1888
1889 const Pipeline * MachEpilogNode::pipeline() const {
1890 return MachNode::pipeline_class();
1891 }
1892
1893 //=============================================================================
1894
1895 static enum RC rc_class(OptoReg::Name reg) {
1896
1897 if (reg == OptoReg::Bad) {
1898 return rc_bad;
1899 }
1900
1901 // we have 32 int registers * 2 halves
1902 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1903
1904 if (reg < slots_of_int_registers) {
1905 return rc_int;
1906 }
1907
1908 // we have 32 float register * 8 halves
1909 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1910 if (reg < slots_of_int_registers + slots_of_float_registers) {
1911 return rc_float;
1912 }
1913
1914 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1915 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1916 return rc_predicate;
1917 }
1918
1919 // Between predicate regs & stack is the flags.
1920 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1921
1922 return rc_stack;
1923 }
1924
1925 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1926 Compile* C = ra_->C;
1927
1928 // Get registers to move.
1929 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1930 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1931 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1932 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1933
1934 enum RC src_hi_rc = rc_class(src_hi);
1935 enum RC src_lo_rc = rc_class(src_lo);
1936 enum RC dst_hi_rc = rc_class(dst_hi);
1937 enum RC dst_lo_rc = rc_class(dst_lo);
1938
1939 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1940
1941 if (src_hi != OptoReg::Bad && !bottom_type()->isa_pvectmask()) {
1942 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1943 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1944 "expected aligned-adjacent pairs");
1945 }
1946
1947 if (src_lo == dst_lo && src_hi == dst_hi) {
1948 return 0; // Self copy, no move.
1949 }
1950
1951 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1952 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1953 int src_offset = ra_->reg2offset(src_lo);
1954 int dst_offset = ra_->reg2offset(dst_lo);
1955
1956 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
1957 uint ireg = ideal_reg();
1958 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
1959 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
1960 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
1961 if (ireg == Op_VecA && masm) {
1962 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1963 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1964 // stack->stack
1965 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1966 sve_vector_reg_size_in_bytes);
1967 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1968 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1969 sve_vector_reg_size_in_bytes);
1970 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1971 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1972 sve_vector_reg_size_in_bytes);
1973 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1974 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1975 as_FloatRegister(Matcher::_regEncode[src_lo]),
1976 as_FloatRegister(Matcher::_regEncode[src_lo]));
1977 } else {
1978 ShouldNotReachHere();
1979 }
1980 } else if (masm) {
1981 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1982 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1983 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1984 // stack->stack
1985 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1986 if (ireg == Op_VecD) {
1987 __ unspill(rscratch1, true, src_offset);
1988 __ spill(rscratch1, true, dst_offset);
1989 } else {
1990 __ spill_copy128(src_offset, dst_offset);
1991 }
1992 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1993 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1994 ireg == Op_VecD ? __ T8B : __ T16B,
1995 as_FloatRegister(Matcher::_regEncode[src_lo]));
1996 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1997 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1998 ireg == Op_VecD ? __ D : __ Q,
1999 ra_->reg2offset(dst_lo));
2000 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2001 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2002 ireg == Op_VecD ? __ D : __ Q,
2003 ra_->reg2offset(src_lo));
2004 } else {
2005 ShouldNotReachHere();
2006 }
2007 }
2008 } else if (masm) {
2009 switch (src_lo_rc) {
2010 case rc_int:
2011 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2012 if (is64) {
2013 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2014 as_Register(Matcher::_regEncode[src_lo]));
2015 } else {
2016 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2017 as_Register(Matcher::_regEncode[src_lo]));
2018 }
2019 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2020 if (is64) {
2021 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2022 as_Register(Matcher::_regEncode[src_lo]));
2023 } else {
2024 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2025 as_Register(Matcher::_regEncode[src_lo]));
2026 }
2027 } else { // gpr --> stack spill
2028 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2029 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2030 }
2031 break;
2032 case rc_float:
2033 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2034 if (is64) {
2035 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2036 as_FloatRegister(Matcher::_regEncode[src_lo]));
2037 } else {
2038 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2039 as_FloatRegister(Matcher::_regEncode[src_lo]));
2040 }
2041 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2042 if (is64) {
2043 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2044 as_FloatRegister(Matcher::_regEncode[src_lo]));
2045 } else {
2046 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2047 as_FloatRegister(Matcher::_regEncode[src_lo]));
2048 }
2049 } else { // fpr --> stack spill
2050 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2051 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2052 is64 ? __ D : __ S, dst_offset);
2053 }
2054 break;
2055 case rc_stack:
2056 if (dst_lo_rc == rc_int) { // stack --> gpr load
2057 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2058 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2059 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2060 is64 ? __ D : __ S, src_offset);
2061 } else if (dst_lo_rc == rc_predicate) {
2062 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2063 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2064 } else { // stack --> stack copy
2065 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2066 if (ideal_reg() == Op_RegVectMask) {
2067 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2068 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2069 } else {
2070 __ unspill(rscratch1, is64, src_offset);
2071 __ spill(rscratch1, is64, dst_offset);
2072 }
2073 }
2074 break;
2075 case rc_predicate:
2076 if (dst_lo_rc == rc_predicate) {
2077 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2078 } else if (dst_lo_rc == rc_stack) {
2079 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2080 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2081 } else {
2082 assert(false, "bad src and dst rc_class combination.");
2083 ShouldNotReachHere();
2084 }
2085 break;
2086 default:
2087 assert(false, "bad rc_class for spill");
2088 ShouldNotReachHere();
2089 }
2090 }
2091
2092 if (st) {
2093 st->print("spill ");
2094 if (src_lo_rc == rc_stack) {
2095 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2096 } else {
2097 st->print("%s -> ", Matcher::regName[src_lo]);
2098 }
2099 if (dst_lo_rc == rc_stack) {
2100 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2101 } else {
2102 st->print("%s", Matcher::regName[dst_lo]);
2103 }
2104 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2105 int vsize = 0;
2106 switch (ideal_reg()) {
2107 case Op_VecD:
2108 vsize = 64;
2109 break;
2110 case Op_VecX:
2111 vsize = 128;
2112 break;
2113 case Op_VecA:
2114 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2115 break;
2116 default:
2117 assert(false, "bad register type for spill");
2118 ShouldNotReachHere();
2119 }
2120 st->print("\t# vector spill size = %d", vsize);
2121 } else if (ideal_reg() == Op_RegVectMask) {
2122 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2123 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2124 st->print("\t# predicate spill size = %d", vsize);
2125 } else {
2126 st->print("\t# spill size = %d", is64 ? 64 : 32);
2127 }
2128 }
2129
2130 return 0;
2131
2132 }
2133
2134 #ifndef PRODUCT
2135 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2136 if (!ra_)
2137 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2138 else
2139 implementation(nullptr, ra_, false, st);
2140 }
2141 #endif
2142
2143 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2144 implementation(masm, ra_, false, nullptr);
2145 }
2146
2147 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2148 return MachNode::size(ra_);
2149 }
2150
2151 //=============================================================================
2152
2153 #ifndef PRODUCT
2154 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2155 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2156 int reg = ra_->get_reg_first(this);
2157 st->print("add %s, rsp, #%d]\t# box lock",
2158 Matcher::regName[reg], offset);
2159 }
2160 #endif
2161
2162 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2163 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2164 int reg = ra_->get_encode(this);
2165
2166 // This add will handle any 24-bit signed offset. 24 bits allows an
2167 // 8 megabyte stack frame.
2168 __ add(as_Register(reg), sp, offset);
2169 }
2170
2171 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2172 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2173 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2174
2175 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2176 return NativeInstruction::instruction_size;
2177 } else {
2178 return 2 * NativeInstruction::instruction_size;
2179 }
2180 }
2181
2182 ///=============================================================================
2183 #ifndef PRODUCT
2184 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2185 {
2186 st->print_cr("# MachVEPNode");
2187 if (!_verified) {
2188 st->print_cr("\t load_class");
2189 } else {
2190 st->print_cr("\t unpack_inline_arg");
2191 }
2192 }
2193 #endif
2194
2195 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2196 {
2197 if (!_verified) {
2198 __ ic_check(1);
2199 } else {
2200 if (ra_->C->stub_function() == nullptr) {
2201 // Emit the entry barrier in a temporary frame before unpacking because
2202 // it can deopt, which would require packing the scalarized args again.
2203 __ verified_entry(ra_->C, 0);
2204 __ entry_barrier();
2205 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2206 __ remove_frame(framesize, false);
2207 }
2208 // Unpack inline type args passed as oop and then jump to
2209 // the verified entry point (skipping the unverified entry).
2210 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2211 // Emit code for verified entry and save increment for stack repair on return
2212 __ verified_entry(ra_->C, sp_inc);
2213 if (Compile::current()->output()->in_scratch_emit_size()) {
2214 Label dummy_verified_entry;
2215 __ b(dummy_verified_entry);
2216 } else {
2217 __ b(*_verified_entry);
2218 }
2219 }
2220 }
2221
2222 //=============================================================================
2223 #ifndef PRODUCT
2224 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2225 {
2226 st->print_cr("# MachUEPNode");
2227 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2228 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2229 st->print_cr("\tcmpw rscratch1, r10");
2230 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2231 }
2232 #endif
2233
2234 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2235 {
2236 __ ic_check(InteriorEntryAlignment);
2237 }
2238
2239 // REQUIRED EMIT CODE
2240
2241 //=============================================================================
2242
2243 // Emit deopt handler code.
2244 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2245 {
2246 // Note that the code buffer's insts_mark is always relative to insts.
2247 // That's why we must use the macroassembler to generate a handler.
2248 address base = __ start_a_stub(size_deopt_handler());
2249 if (base == nullptr) {
2250 ciEnv::current()->record_failure("CodeCache is full");
2251 return 0; // CodeBuffer::expand failed
2252 }
2253
2254 int offset = __ offset();
2255 Label start;
2256 __ bind(start);
2257 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2258
2259 int entry_offset = __ offset();
2260 __ b(start);
2261
2262 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2263 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2264 "out of bounds read in post-call NOP check");
2265 __ end_a_stub();
2266 return entry_offset;
2267 }
2268
2269 // REQUIRED MATCHER CODE
2270
2271 //=============================================================================
2272
2273 bool Matcher::match_rule_supported(int opcode) {
2274 if (!has_match_rule(opcode))
2275 return false;
2276
2277 switch (opcode) {
2278 case Op_OnSpinWait:
2279 return VM_Version::supports_on_spin_wait();
2280 case Op_CacheWB:
2281 case Op_CacheWBPreSync:
2282 case Op_CacheWBPostSync:
2283 if (!VM_Version::supports_data_cache_line_flush()) {
2284 return false;
2285 }
2286 break;
2287 case Op_ExpandBits:
2288 case Op_CompressBits:
2289 if (!VM_Version::supports_svebitperm()) {
2290 return false;
2291 }
2292 break;
2293 case Op_FmaF:
2294 case Op_FmaD:
2295 case Op_FmaVF:
2296 case Op_FmaVD:
2297 if (!UseFMA) {
2298 return false;
2299 }
2300 break;
2301 case Op_FmaHF:
2302 // UseFMA flag also needs to be checked along with FEAT_FP16
2303 if (!UseFMA || !is_feat_fp16_supported()) {
2304 return false;
2305 }
2306 break;
2307 case Op_AddHF:
2308 case Op_SubHF:
2309 case Op_MulHF:
2310 case Op_DivHF:
2311 case Op_MinHF:
2312 case Op_MaxHF:
2313 case Op_SqrtHF:
2314 // Half-precision floating point scalar operations require FEAT_FP16
2315 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2316 // features are supported.
2317 if (!is_feat_fp16_supported()) {
2318 return false;
2319 }
2320 break;
2321 }
2322
2323 return true; // Per default match rules are supported.
2324 }
2325
2326 const RegMask* Matcher::predicate_reg_mask(void) {
2327 return &_PR_REG_mask;
2328 }
2329
2330 bool Matcher::supports_vector_calling_convention(void) {
2331 return EnableVectorSupport;
2332 }
2333
2334 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2335 assert(EnableVectorSupport, "sanity");
2336 int lo = V0_num;
2337 int hi = V0_H_num;
2338 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2339 hi = V0_K_num;
2340 }
2341 return OptoRegPair(hi, lo);
2342 }
2343
2344 // Is this branch offset short enough that a short branch can be used?
2345 //
2346 // NOTE: If the platform does not provide any short branch variants, then
2347 // this method should return false for offset 0.
2348 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2349 // The passed offset is relative to address of the branch.
2350
2351 return (-32768 <= offset && offset < 32768);
2352 }
2353
2354 // Vector width in bytes.
2355 int Matcher::vector_width_in_bytes(BasicType bt) {
2356 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2357 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2358 // Minimum 2 values in vector
2359 if (size < 2*type2aelembytes(bt)) size = 0;
2360 // But never < 4
2361 if (size < 4) size = 0;
2362 return size;
2363 }
2364
2365 // Limits on vector size (number of elements) loaded into vector.
2366 int Matcher::max_vector_size(const BasicType bt) {
2367 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2368 }
2369
2370 int Matcher::min_vector_size(const BasicType bt) {
2371 // Usually, the shortest vector length supported by AArch64 ISA and
2372 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2373 // vectors in a few special cases.
2374 int size;
2375 switch(bt) {
2376 case T_BOOLEAN:
2377 // Load/store a vector mask with only 2 elements for vector types
2378 // such as "2I/2F/2L/2D".
2379 size = 2;
2380 break;
2381 case T_BYTE:
2382 // Generate a "4B" vector, to support vector cast between "8B/16B"
2383 // and "4S/4I/4L/4F/4D".
2384 size = 4;
2385 break;
2386 case T_SHORT:
2387 // Generate a "2S" vector, to support vector cast between "4S/8S"
2388 // and "2I/2L/2F/2D".
2389 size = 2;
2390 break;
2391 default:
2392 // Limit the min vector length to 64-bit.
2393 size = 8 / type2aelembytes(bt);
2394 // The number of elements in a vector should be at least 2.
2395 size = MAX2(size, 2);
2396 }
2397
2398 int max_size = max_vector_size(bt);
2399 return MIN2(size, max_size);
2400 }
2401
2402 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2403 return Matcher::max_vector_size(bt);
2404 }
2405
2406 // Actual max scalable vector register length.
2407 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2408 return Matcher::max_vector_size(bt);
2409 }
2410
2411 // Vector ideal reg.
2412 uint Matcher::vector_ideal_reg(int len) {
2413 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2414 return Op_VecA;
2415 }
2416 switch(len) {
2417 // For 16-bit/32-bit mask vector, reuse VecD.
2418 case 2:
2419 case 4:
2420 case 8: return Op_VecD;
2421 case 16: return Op_VecX;
2422 }
2423 ShouldNotReachHere();
2424 return 0;
2425 }
2426
2427 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2428 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2429 switch (ideal_reg) {
2430 case Op_VecA: return new vecAOper();
2431 case Op_VecD: return new vecDOper();
2432 case Op_VecX: return new vecXOper();
2433 }
2434 ShouldNotReachHere();
2435 return nullptr;
2436 }
2437
2438 bool Matcher::is_reg2reg_move(MachNode* m) {
2439 return false;
2440 }
2441
2442 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2443 return false;
2444 }
2445
2446 bool Matcher::is_generic_vector(MachOper* opnd) {
2447 return opnd->opcode() == VREG;
2448 }
2449
2450 #ifdef ASSERT
2451 // Return whether or not this register is ever used as an argument.
2452 bool Matcher::can_be_java_arg(int reg)
2453 {
2454 return
2455 reg == R0_num || reg == R0_H_num ||
2456 reg == R1_num || reg == R1_H_num ||
2457 reg == R2_num || reg == R2_H_num ||
2458 reg == R3_num || reg == R3_H_num ||
2459 reg == R4_num || reg == R4_H_num ||
2460 reg == R5_num || reg == R5_H_num ||
2461 reg == R6_num || reg == R6_H_num ||
2462 reg == R7_num || reg == R7_H_num ||
2463 reg == V0_num || reg == V0_H_num ||
2464 reg == V1_num || reg == V1_H_num ||
2465 reg == V2_num || reg == V2_H_num ||
2466 reg == V3_num || reg == V3_H_num ||
2467 reg == V4_num || reg == V4_H_num ||
2468 reg == V5_num || reg == V5_H_num ||
2469 reg == V6_num || reg == V6_H_num ||
2470 reg == V7_num || reg == V7_H_num;
2471 }
2472 #endif
2473
2474 uint Matcher::int_pressure_limit()
2475 {
2476 // JDK-8183543: When taking the number of available registers as int
2477 // register pressure threshold, the jtreg test:
2478 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2479 // failed due to C2 compilation failure with
2480 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2481 //
2482 // A derived pointer is live at CallNode and then is flagged by RA
2483 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2484 // derived pointers and lastly fail to spill after reaching maximum
2485 // number of iterations. Lowering the default pressure threshold to
2486 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2487 // a high register pressure area of the code so that split_DEF can
2488 // generate DefinitionSpillCopy for the derived pointer.
2489 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2490 if (!PreserveFramePointer) {
2491 // When PreserveFramePointer is off, frame pointer is allocatable,
2492 // but different from other SOC registers, it is excluded from
2493 // fatproj's mask because its save type is No-Save. Decrease 1 to
2494 // ensure high pressure at fatproj when PreserveFramePointer is off.
2495 // See check_pressure_at_fatproj().
2496 default_int_pressure_threshold--;
2497 }
2498 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2499 }
2500
2501 uint Matcher::float_pressure_limit()
2502 {
2503 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2504 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2505 }
2506
2507 const RegMask& Matcher::divI_proj_mask() {
2508 ShouldNotReachHere();
2509 return RegMask::EMPTY;
2510 }
2511
2512 // Register for MODI projection of divmodI.
2513 const RegMask& Matcher::modI_proj_mask() {
2514 ShouldNotReachHere();
2515 return RegMask::EMPTY;
2516 }
2517
2518 // Register for DIVL projection of divmodL.
2519 const RegMask& Matcher::divL_proj_mask() {
2520 ShouldNotReachHere();
2521 return RegMask::EMPTY;
2522 }
2523
2524 // Register for MODL projection of divmodL.
2525 const RegMask& Matcher::modL_proj_mask() {
2526 ShouldNotReachHere();
2527 return RegMask::EMPTY;
2528 }
2529
2530 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2531 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2532 Node* u = addp->fast_out(i);
2533 if (u->is_LoadStore()) {
2534 // On AArch64, LoadStoreNodes (i.e. compare and swap
2535 // instructions) only take register indirect as an operand, so
2536 // any attempt to use an AddPNode as an input to a LoadStoreNode
2537 // must fail.
2538 return false;
2539 }
2540 if (u->is_Mem()) {
2541 int opsize = u->as_Mem()->memory_size();
2542 assert(opsize > 0, "unexpected memory operand size");
2543 if (u->as_Mem()->memory_size() != (1<<shift)) {
2544 return false;
2545 }
2546 }
2547 }
2548 return true;
2549 }
2550
2551 // Convert BoolTest condition to Assembler condition.
2552 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2553 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2554 Assembler::Condition result;
2555 switch(cond) {
2556 case BoolTest::eq:
2557 result = Assembler::EQ; break;
2558 case BoolTest::ne:
2559 result = Assembler::NE; break;
2560 case BoolTest::le:
2561 result = Assembler::LE; break;
2562 case BoolTest::ge:
2563 result = Assembler::GE; break;
2564 case BoolTest::lt:
2565 result = Assembler::LT; break;
2566 case BoolTest::gt:
2567 result = Assembler::GT; break;
2568 case BoolTest::ule:
2569 result = Assembler::LS; break;
2570 case BoolTest::uge:
2571 result = Assembler::HS; break;
2572 case BoolTest::ult:
2573 result = Assembler::LO; break;
2574 case BoolTest::ugt:
2575 result = Assembler::HI; break;
2576 case BoolTest::overflow:
2577 result = Assembler::VS; break;
2578 case BoolTest::no_overflow:
2579 result = Assembler::VC; break;
2580 default:
2581 ShouldNotReachHere();
2582 return Assembler::Condition(-1);
2583 }
2584
2585 // Check conversion
2586 if (cond & BoolTest::unsigned_compare) {
2587 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2588 } else {
2589 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2590 }
2591
2592 return result;
2593 }
2594
2595 // Binary src (Replicate con)
2596 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2597 if (n == nullptr || m == nullptr) {
2598 return false;
2599 }
2600
2601 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2602 return false;
2603 }
2604
2605 Node* imm_node = m->in(1);
2606 if (!imm_node->is_Con()) {
2607 return false;
2608 }
2609
2610 const Type* t = imm_node->bottom_type();
2611 if (!(t->isa_int() || t->isa_long())) {
2612 return false;
2613 }
2614
2615 switch (n->Opcode()) {
2616 case Op_AndV:
2617 case Op_OrV:
2618 case Op_XorV: {
2619 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2620 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2621 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2622 }
2623 case Op_AddVB:
2624 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2625 case Op_AddVS:
2626 case Op_AddVI:
2627 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2628 case Op_AddVL:
2629 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2630 default:
2631 return false;
2632 }
2633 }
2634
2635 // (XorV src (Replicate m1))
2636 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2637 if (n != nullptr && m != nullptr) {
2638 return n->Opcode() == Op_XorV &&
2639 VectorNode::is_all_ones_vector(m);
2640 }
2641 return false;
2642 }
2643
2644 // Returns true if (n, m) matches "(XorVMask vm2 (MaskAll m1))" and that XorVMask
2645 // is used only by an AndVMask. In that case, cloning m (the MaskAll) lets the
2646 // matcher avoid sharing the MaskAll node and subsume the pattern into rule:
2647 // "(AndVMask vm1 (XorVMask vm2 (MaskAll m1)))".
2648 //
2649 // Limitation: the "andNot" rule still cannot be matched if "m" has other
2650 // uses outside this pattern.
2651 static bool is_vector_mask_not_operand_in_andnot_pattern(Node* n, Node* m) {
2652 if (n == nullptr || m == nullptr) {
2653 return false;
2654 }
2655
2656 if (VectorNode::is_all_ones_vector(m) &&
2657 n->Opcode() == Op_XorVMask &&
2658 n->outcnt() == 1 &&
2659 n->unique_out()->Opcode() == Op_AndVMask) {
2660 // If another input of the AndVMask is also a mask-not pattern that would
2661 // qualify for the `maskAll` cloning, do not clone the "maskAll" here,
2662 // because the match rule can only consume one such pattern.
2663 Node* use = n->unique_out();
2664 Node* other_input = use->in(1) == n ? use->in(2) : use->in(1);
2665 return !VectorNode::is_vectormask_bitwise_not_pattern(other_input);
2666 }
2667 return false;
2668 }
2669
2670 // Should the matcher clone input 'm' of node 'n'?
2671 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2672 if (is_vshift_con_pattern(n, m) ||
2673 is_vector_bitwise_not_pattern(n, m) ||
2674 is_vector_mask_not_operand_in_andnot_pattern(n, m) ||
2675 is_valid_sve_arith_imm_pattern(n, m) ||
2676 is_encode_and_store_pattern(n, m)) {
2677 mstack.push(m, Visit);
2678 return true;
2679 }
2680 return false;
2681 }
2682
2683 // Should the Matcher clone shifts on addressing modes, expecting them
2684 // to be subsumed into complex addressing expressions or compute them
2685 // into registers?
2686 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2687
2688 // Loads and stores with indirect memory input (e.g., volatile loads and
2689 // stores) do not subsume the input into complex addressing expressions. If
2690 // the addressing expression is input to at least one such load or store, do
2691 // not clone the addressing expression. Query needs_acquiring_load and
2692 // needs_releasing_store as a proxy for indirect memory input, as it is not
2693 // possible to directly query for indirect memory input at this stage.
2694 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2695 Node* n = m->fast_out(i);
2696 if (n->is_Load() && needs_acquiring_load(n)) {
2697 return false;
2698 }
2699 if (n->is_Store() && needs_releasing_store(n)) {
2700 return false;
2701 }
2702 }
2703
2704 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2705 return true;
2706 }
2707
2708 Node *off = m->in(AddPNode::Offset);
2709 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2710 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2711 // Are there other uses besides address expressions?
2712 !is_visited(off)) {
2713 address_visited.set(off->_idx); // Flag as address_visited
2714 mstack.push(off->in(2), Visit);
2715 Node *conv = off->in(1);
2716 if (conv->Opcode() == Op_ConvI2L &&
2717 // Are there other uses besides address expressions?
2718 !is_visited(conv)) {
2719 address_visited.set(conv->_idx); // Flag as address_visited
2720 mstack.push(conv->in(1), Pre_Visit);
2721 } else {
2722 mstack.push(conv, Pre_Visit);
2723 }
2724 address_visited.test_set(m->_idx); // Flag as address_visited
2725 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2726 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2727 return true;
2728 } else if (off->Opcode() == Op_ConvI2L &&
2729 // Are there other uses besides address expressions?
2730 !is_visited(off)) {
2731 address_visited.test_set(m->_idx); // Flag as address_visited
2732 address_visited.set(off->_idx); // Flag as address_visited
2733 mstack.push(off->in(1), Pre_Visit);
2734 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2735 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2736 return true;
2737 }
2738 return false;
2739 }
2740
2741 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2742 { \
2743 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2744 guarantee(DISP == 0, "mode not permitted for volatile"); \
2745 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2746 __ INSN(REG, as_Register(BASE)); \
2747 }
2748
2749
2750 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2751 {
2752 Address::extend scale;
2753
2754 // Hooboy, this is fugly. We need a way to communicate to the
2755 // encoder that the index needs to be sign extended, so we have to
2756 // enumerate all the cases.
2757 switch (opcode) {
2758 case INDINDEXSCALEDI2L:
2759 case INDINDEXSCALEDI2LN:
2760 case INDINDEXI2L:
2761 case INDINDEXI2LN:
2762 scale = Address::sxtw(size);
2763 break;
2764 default:
2765 scale = Address::lsl(size);
2766 }
2767
2768 if (index == -1) {
2769 return Address(base, disp);
2770 } else {
2771 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2772 return Address(base, as_Register(index), scale);
2773 }
2774 }
2775
2776
2777 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2778 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2779 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2780 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2781 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2782
2783 // Used for all non-volatile memory accesses. The use of
2784 // $mem->opcode() to discover whether this pattern uses sign-extended
2785 // offsets is something of a kludge.
2786 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2787 Register reg, int opcode,
2788 Register base, int index, int scale, int disp,
2789 int size_in_memory)
2790 {
2791 Address addr = mem2address(opcode, base, index, scale, disp);
2792 if (addr.getMode() == Address::base_plus_offset) {
2793 /* Fix up any out-of-range offsets. */
2794 assert_different_registers(rscratch1, base);
2795 assert_different_registers(rscratch1, reg);
2796 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2797 }
2798 (masm->*insn)(reg, addr);
2799 }
2800
2801 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2802 FloatRegister reg, int opcode,
2803 Register base, int index, int size, int disp,
2804 int size_in_memory)
2805 {
2806 Address::extend scale;
2807
2808 switch (opcode) {
2809 case INDINDEXSCALEDI2L:
2810 case INDINDEXSCALEDI2LN:
2811 scale = Address::sxtw(size);
2812 break;
2813 default:
2814 scale = Address::lsl(size);
2815 }
2816
2817 if (index == -1) {
2818 // Fix up any out-of-range offsets.
2819 assert_different_registers(rscratch1, base);
2820 Address addr = Address(base, disp);
2821 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2822 (masm->*insn)(reg, addr);
2823 } else {
2824 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2825 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2826 }
2827 }
2828
2829 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2830 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2831 int opcode, Register base, int index, int size, int disp)
2832 {
2833 if (index == -1) {
2834 (masm->*insn)(reg, T, Address(base, disp));
2835 } else {
2836 assert(disp == 0, "unsupported address mode");
2837 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2838 }
2839 }
2840
2841 %}
2842
2843
2844
2845 //----------ENCODING BLOCK-----------------------------------------------------
2846 // This block specifies the encoding classes used by the compiler to
2847 // output byte streams. Encoding classes are parameterized macros
2848 // used by Machine Instruction Nodes in order to generate the bit
2849 // encoding of the instruction. Operands specify their base encoding
2850 // interface with the interface keyword. There are currently
2851 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2852 // COND_INTER. REG_INTER causes an operand to generate a function
2853 // which returns its register number when queried. CONST_INTER causes
2854 // an operand to generate a function which returns the value of the
2855 // constant when queried. MEMORY_INTER causes an operand to generate
2856 // four functions which return the Base Register, the Index Register,
2857 // the Scale Value, and the Offset Value of the operand when queried.
2858 // COND_INTER causes an operand to generate six functions which return
2859 // the encoding code (ie - encoding bits for the instruction)
2860 // associated with each basic boolean condition for a conditional
2861 // instruction.
2862 //
2863 // Instructions specify two basic values for encoding. Again, a
2864 // function is available to check if the constant displacement is an
2865 // oop. They use the ins_encode keyword to specify their encoding
2866 // classes (which must be a sequence of enc_class names, and their
2867 // parameters, specified in the encoding block), and they use the
2868 // opcode keyword to specify, in order, their primary, secondary, and
2869 // tertiary opcode. Only the opcode sections which a particular
2870 // instruction needs for encoding need to be specified.
2871 encode %{
2872 // Build emit functions for each basic byte or larger field in the
2873 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2874 // from C++ code in the enc_class source block. Emit functions will
2875 // live in the main source block for now. In future, we can
2876 // generalize this by adding a syntax that specifies the sizes of
2877 // fields in an order, so that the adlc can build the emit functions
2878 // automagically
2879
2880 // catch all for unimplemented encodings
2881 enc_class enc_unimplemented %{
2882 __ unimplemented("C2 catch all");
2883 %}
2884
2885 // BEGIN Non-volatile memory access
2886
2887 // This encoding class is generated automatically from ad_encode.m4.
2888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2889 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2890 Register dst_reg = as_Register($dst$$reg);
2891 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2892 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2893 %}
2894
2895 // This encoding class is generated automatically from ad_encode.m4.
2896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2897 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2898 Register dst_reg = as_Register($dst$$reg);
2899 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2900 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2901 %}
2902
2903 // This encoding class is generated automatically from ad_encode.m4.
2904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2905 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2906 Register dst_reg = as_Register($dst$$reg);
2907 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2908 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2909 %}
2910
2911 // This encoding class is generated automatically from ad_encode.m4.
2912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2913 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2914 Register dst_reg = as_Register($dst$$reg);
2915 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2916 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2917 %}
2918
2919 // This encoding class is generated automatically from ad_encode.m4.
2920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2921 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2922 Register dst_reg = as_Register($dst$$reg);
2923 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2924 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2925 %}
2926
2927 // This encoding class is generated automatically from ad_encode.m4.
2928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2929 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2930 Register dst_reg = as_Register($dst$$reg);
2931 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2932 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2933 %}
2934
2935 // This encoding class is generated automatically from ad_encode.m4.
2936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2937 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2938 Register dst_reg = as_Register($dst$$reg);
2939 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2940 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2941 %}
2942
2943 // This encoding class is generated automatically from ad_encode.m4.
2944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2945 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2946 Register dst_reg = as_Register($dst$$reg);
2947 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2948 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2949 %}
2950
2951 // This encoding class is generated automatically from ad_encode.m4.
2952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2953 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2954 Register dst_reg = as_Register($dst$$reg);
2955 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2956 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2957 %}
2958
2959 // This encoding class is generated automatically from ad_encode.m4.
2960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2961 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2962 Register dst_reg = as_Register($dst$$reg);
2963 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2964 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2965 %}
2966
2967 // This encoding class is generated automatically from ad_encode.m4.
2968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2969 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2970 Register dst_reg = as_Register($dst$$reg);
2971 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2972 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2973 %}
2974
2975 // This encoding class is generated automatically from ad_encode.m4.
2976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2977 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2978 Register dst_reg = as_Register($dst$$reg);
2979 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2980 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2981 %}
2982
2983 // This encoding class is generated automatically from ad_encode.m4.
2984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2985 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2986 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2987 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2988 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2989 %}
2990
2991 // This encoding class is generated automatically from ad_encode.m4.
2992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2993 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2994 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2995 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2996 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2997 %}
2998
2999 // This encoding class is generated automatically from ad_encode.m4.
3000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3001 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3002 Register src_reg = as_Register($src$$reg);
3003 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3004 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3005 %}
3006
3007 // This encoding class is generated automatically from ad_encode.m4.
3008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3009 enc_class aarch64_enc_strb0(memory1 mem) %{
3010 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3011 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3012 %}
3013
3014 // This encoding class is generated automatically from ad_encode.m4.
3015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3016 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3017 Register src_reg = as_Register($src$$reg);
3018 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3019 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3020 %}
3021
3022 // This encoding class is generated automatically from ad_encode.m4.
3023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3024 enc_class aarch64_enc_strh0(memory2 mem) %{
3025 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3026 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3027 %}
3028
3029 // This encoding class is generated automatically from ad_encode.m4.
3030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3031 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3032 Register src_reg = as_Register($src$$reg);
3033 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3034 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3035 %}
3036
3037 // This encoding class is generated automatically from ad_encode.m4.
3038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3039 enc_class aarch64_enc_strw0(memory4 mem) %{
3040 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3041 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3042 %}
3043
3044 // This encoding class is generated automatically from ad_encode.m4.
3045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3046 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3047 Register src_reg = as_Register($src$$reg);
3048 // we sometimes get asked to store the stack pointer into the
3049 // current thread -- we cannot do that directly on AArch64
3050 if (src_reg == r31_sp) {
3051 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3052 __ mov(rscratch2, sp);
3053 src_reg = rscratch2;
3054 }
3055 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3056 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3057 %}
3058
3059 // This encoding class is generated automatically from ad_encode.m4.
3060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3061 enc_class aarch64_enc_str0(memory8 mem) %{
3062 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3063 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3064 %}
3065
3066 // This encoding class is generated automatically from ad_encode.m4.
3067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3068 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3069 FloatRegister src_reg = as_FloatRegister($src$$reg);
3070 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3071 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3072 %}
3073
3074 // This encoding class is generated automatically from ad_encode.m4.
3075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3076 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3077 FloatRegister src_reg = as_FloatRegister($src$$reg);
3078 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3079 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3080 %}
3081
3082 // This encoding class is generated automatically from ad_encode.m4.
3083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3084 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3085 __ membar(Assembler::StoreStore);
3086 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3087 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3088 %}
3089
3090 // END Non-volatile memory access
3091
3092 // Vector loads and stores
3093 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3094 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3095 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3096 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3097 %}
3098
3099 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3100 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3101 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3102 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3103 %}
3104
3105 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3106 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3107 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3108 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3109 %}
3110
3111 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3112 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3113 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3114 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3115 %}
3116
3117 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3118 FloatRegister src_reg = as_FloatRegister($src$$reg);
3119 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3120 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3121 %}
3122
3123 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3124 FloatRegister src_reg = as_FloatRegister($src$$reg);
3125 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3126 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3127 %}
3128
3129 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3130 FloatRegister src_reg = as_FloatRegister($src$$reg);
3131 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3132 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3133 %}
3134
3135 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3136 FloatRegister src_reg = as_FloatRegister($src$$reg);
3137 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3138 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3139 %}
3140
3141 // volatile loads and stores
3142
3143 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3144 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3145 rscratch1, stlrb);
3146 %}
3147
3148 enc_class aarch64_enc_stlrb0(memory mem) %{
3149 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3150 rscratch1, stlrb);
3151 %}
3152
3153 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3154 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, stlrh);
3156 %}
3157
3158 enc_class aarch64_enc_stlrh0(memory mem) %{
3159 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, stlrh);
3161 %}
3162
3163 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3164 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3165 rscratch1, stlrw);
3166 %}
3167
3168 enc_class aarch64_enc_stlrw0(memory mem) %{
3169 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3170 rscratch1, stlrw);
3171 %}
3172
3173 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3174 Register dst_reg = as_Register($dst$$reg);
3175 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, ldarb);
3177 __ sxtbw(dst_reg, dst_reg);
3178 %}
3179
3180 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3181 Register dst_reg = as_Register($dst$$reg);
3182 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3183 rscratch1, ldarb);
3184 __ sxtb(dst_reg, dst_reg);
3185 %}
3186
3187 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3188 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, ldarb);
3190 %}
3191
3192 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3193 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, ldarb);
3195 %}
3196
3197 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3198 Register dst_reg = as_Register($dst$$reg);
3199 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3200 rscratch1, ldarh);
3201 __ sxthw(dst_reg, dst_reg);
3202 %}
3203
3204 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3205 Register dst_reg = as_Register($dst$$reg);
3206 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3207 rscratch1, ldarh);
3208 __ sxth(dst_reg, dst_reg);
3209 %}
3210
3211 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3212 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3213 rscratch1, ldarh);
3214 %}
3215
3216 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3217 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3218 rscratch1, ldarh);
3219 %}
3220
3221 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3222 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3223 rscratch1, ldarw);
3224 %}
3225
3226 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3227 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3228 rscratch1, ldarw);
3229 %}
3230
3231 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3232 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3233 rscratch1, ldar);
3234 %}
3235
3236 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3237 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3238 rscratch1, ldarw);
3239 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3240 %}
3241
3242 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3243 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3244 rscratch1, ldar);
3245 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3246 %}
3247
3248 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3249 Register src_reg = as_Register($src$$reg);
3250 // we sometimes get asked to store the stack pointer into the
3251 // current thread -- we cannot do that directly on AArch64
3252 if (src_reg == r31_sp) {
3253 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3254 __ mov(rscratch2, sp);
3255 src_reg = rscratch2;
3256 }
3257 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3258 rscratch1, stlr);
3259 %}
3260
3261 enc_class aarch64_enc_stlr0(memory mem) %{
3262 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3263 rscratch1, stlr);
3264 %}
3265
3266 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3267 {
3268 FloatRegister src_reg = as_FloatRegister($src$$reg);
3269 __ fmovs(rscratch2, src_reg);
3270 }
3271 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3272 rscratch1, stlrw);
3273 %}
3274
3275 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3276 {
3277 FloatRegister src_reg = as_FloatRegister($src$$reg);
3278 __ fmovd(rscratch2, src_reg);
3279 }
3280 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3281 rscratch1, stlr);
3282 %}
3283
3284 // synchronized read/update encodings
3285
3286 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3287 Register dst_reg = as_Register($dst$$reg);
3288 Register base = as_Register($mem$$base);
3289 int index = $mem$$index;
3290 int scale = $mem$$scale;
3291 int disp = $mem$$disp;
3292 if (index == -1) {
3293 if (disp != 0) {
3294 __ lea(rscratch1, Address(base, disp));
3295 __ ldaxr(dst_reg, rscratch1);
3296 } else {
3297 // TODO
3298 // should we ever get anything other than this case?
3299 __ ldaxr(dst_reg, base);
3300 }
3301 } else {
3302 Register index_reg = as_Register(index);
3303 if (disp == 0) {
3304 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3305 __ ldaxr(dst_reg, rscratch1);
3306 } else {
3307 __ lea(rscratch1, Address(base, disp));
3308 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3309 __ ldaxr(dst_reg, rscratch1);
3310 }
3311 }
3312 %}
3313
3314 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3315 Register src_reg = as_Register($src$$reg);
3316 Register base = as_Register($mem$$base);
3317 int index = $mem$$index;
3318 int scale = $mem$$scale;
3319 int disp = $mem$$disp;
3320 if (index == -1) {
3321 if (disp != 0) {
3322 __ lea(rscratch2, Address(base, disp));
3323 __ stlxr(rscratch1, src_reg, rscratch2);
3324 } else {
3325 // TODO
3326 // should we ever get anything other than this case?
3327 __ stlxr(rscratch1, src_reg, base);
3328 }
3329 } else {
3330 Register index_reg = as_Register(index);
3331 if (disp == 0) {
3332 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3333 __ stlxr(rscratch1, src_reg, rscratch2);
3334 } else {
3335 __ lea(rscratch2, Address(base, disp));
3336 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3337 __ stlxr(rscratch1, src_reg, rscratch2);
3338 }
3339 }
3340 __ cmpw(rscratch1, zr);
3341 %}
3342
3343 // prefetch encodings
3344
3345 enc_class aarch64_enc_prefetchw(memory mem) %{
3346 Register base = as_Register($mem$$base);
3347 int index = $mem$$index;
3348 int scale = $mem$$scale;
3349 int disp = $mem$$disp;
3350 if (index == -1) {
3351 // Fix up any out-of-range offsets.
3352 assert_different_registers(rscratch1, base);
3353 Address addr = Address(base, disp);
3354 addr = __ legitimize_address(addr, 8, rscratch1);
3355 __ prfm(addr, PSTL1KEEP);
3356 } else {
3357 Register index_reg = as_Register(index);
3358 if (disp == 0) {
3359 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3360 } else {
3361 __ lea(rscratch1, Address(base, disp));
3362 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3363 }
3364 }
3365 %}
3366
3367 // mov encodings
3368
3369 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3370 uint32_t con = (uint32_t)$src$$constant;
3371 Register dst_reg = as_Register($dst$$reg);
3372 if (con == 0) {
3373 __ movw(dst_reg, zr);
3374 } else {
3375 __ movw(dst_reg, con);
3376 }
3377 %}
3378
3379 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3380 Register dst_reg = as_Register($dst$$reg);
3381 uint64_t con = (uint64_t)$src$$constant;
3382 if (con == 0) {
3383 __ mov(dst_reg, zr);
3384 } else {
3385 __ mov(dst_reg, con);
3386 }
3387 %}
3388
3389 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3390 Register dst_reg = as_Register($dst$$reg);
3391 address con = (address)$src$$constant;
3392 if (con == nullptr || con == (address)1) {
3393 ShouldNotReachHere();
3394 } else {
3395 relocInfo::relocType rtype = $src->constant_reloc();
3396 if (rtype == relocInfo::oop_type) {
3397 __ movoop(dst_reg, (jobject)con);
3398 } else if (rtype == relocInfo::metadata_type) {
3399 __ mov_metadata(dst_reg, (Metadata*)con);
3400 } else {
3401 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3402 // load fake address constants using a normal move
3403 if (! __ is_valid_AArch64_address(con) ||
3404 con < (address)(uintptr_t)os::vm_page_size() ||
3405 rtype == relocInfo::none) {
3406 __ mov(dst_reg, con);
3407 } else {
3408 // use shorter adrp/add sequence for external_word relocation
3409 uint64_t offset;
3410 __ adrp(dst_reg, Address(con, rtype), offset);
3411 __ add(dst_reg, dst_reg, offset);
3412 }
3413 }
3414 }
3415 %}
3416
3417 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3418 Register dst_reg = as_Register($dst$$reg);
3419 __ mov(dst_reg, zr);
3420 %}
3421
3422 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3423 Register dst_reg = as_Register($dst$$reg);
3424 __ mov(dst_reg, (uint64_t)1);
3425 %}
3426
3427 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3428 Register dst_reg = as_Register($dst$$reg);
3429 address con = (address)$src$$constant;
3430 if (con == nullptr) {
3431 ShouldNotReachHere();
3432 } else {
3433 relocInfo::relocType rtype = $src->constant_reloc();
3434 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3435 __ set_narrow_oop(dst_reg, (jobject)con);
3436 }
3437 %}
3438
3439 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3440 Register dst_reg = as_Register($dst$$reg);
3441 __ mov(dst_reg, zr);
3442 %}
3443
3444 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3445 Register dst_reg = as_Register($dst$$reg);
3446 address con = (address)$src$$constant;
3447 if (con == nullptr) {
3448 ShouldNotReachHere();
3449 } else {
3450 relocInfo::relocType rtype = $src->constant_reloc();
3451 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3452 __ set_narrow_klass(dst_reg, (Klass *)con);
3453 }
3454 %}
3455
3456 // arithmetic encodings
3457
3458 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3459 Register dst_reg = as_Register($dst$$reg);
3460 Register src_reg = as_Register($src1$$reg);
3461 int32_t con = (int32_t)$src2$$constant;
3462 // add has primary == 0, subtract has primary == 1
3463 if ($primary) { con = -con; }
3464 if (con < 0) {
3465 __ subw(dst_reg, src_reg, -con);
3466 } else {
3467 __ addw(dst_reg, src_reg, con);
3468 }
3469 %}
3470
3471 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3472 Register dst_reg = as_Register($dst$$reg);
3473 Register src_reg = as_Register($src1$$reg);
3474 int32_t con = (int32_t)$src2$$constant;
3475 // add has primary == 0, subtract has primary == 1
3476 if ($primary) { con = -con; }
3477 if (con < 0) {
3478 __ sub(dst_reg, src_reg, -con);
3479 } else {
3480 __ add(dst_reg, src_reg, con);
3481 }
3482 %}
3483
3484 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3485 Register dst_reg = as_Register($dst$$reg);
3486 Register src1_reg = as_Register($src1$$reg);
3487 Register src2_reg = as_Register($src2$$reg);
3488 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3489 %}
3490
3491 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3492 Register dst_reg = as_Register($dst$$reg);
3493 Register src1_reg = as_Register($src1$$reg);
3494 Register src2_reg = as_Register($src2$$reg);
3495 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3496 %}
3497
3498 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3499 Register dst_reg = as_Register($dst$$reg);
3500 Register src1_reg = as_Register($src1$$reg);
3501 Register src2_reg = as_Register($src2$$reg);
3502 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3503 %}
3504
3505 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3506 Register dst_reg = as_Register($dst$$reg);
3507 Register src1_reg = as_Register($src1$$reg);
3508 Register src2_reg = as_Register($src2$$reg);
3509 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3510 %}
3511
3512 // compare instruction encodings
3513
3514 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3515 Register reg1 = as_Register($src1$$reg);
3516 Register reg2 = as_Register($src2$$reg);
3517 __ cmpw(reg1, reg2);
3518 %}
3519
3520 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3521 Register reg = as_Register($src1$$reg);
3522 int32_t val = $src2$$constant;
3523 if (val >= 0) {
3524 __ subsw(zr, reg, val);
3525 } else {
3526 __ addsw(zr, reg, -val);
3527 }
3528 %}
3529
3530 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3531 Register reg1 = as_Register($src1$$reg);
3532 uint32_t val = (uint32_t)$src2$$constant;
3533 __ movw(rscratch1, val);
3534 __ cmpw(reg1, rscratch1);
3535 %}
3536
3537 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3538 Register reg1 = as_Register($src1$$reg);
3539 Register reg2 = as_Register($src2$$reg);
3540 __ cmp(reg1, reg2);
3541 %}
3542
3543 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3544 Register reg = as_Register($src1$$reg);
3545 int64_t val = $src2$$constant;
3546 if (val >= 0) {
3547 __ subs(zr, reg, val);
3548 } else if (val != -val) {
3549 __ adds(zr, reg, -val);
3550 } else {
3551 // aargh, Long.MIN_VALUE is a special case
3552 __ orr(rscratch1, zr, (uint64_t)val);
3553 __ subs(zr, reg, rscratch1);
3554 }
3555 %}
3556
3557 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3558 Register reg1 = as_Register($src1$$reg);
3559 uint64_t val = (uint64_t)$src2$$constant;
3560 __ mov(rscratch1, val);
3561 __ cmp(reg1, rscratch1);
3562 %}
3563
3564 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3565 Register reg1 = as_Register($src1$$reg);
3566 Register reg2 = as_Register($src2$$reg);
3567 __ cmp(reg1, reg2);
3568 %}
3569
3570 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3571 Register reg1 = as_Register($src1$$reg);
3572 Register reg2 = as_Register($src2$$reg);
3573 __ cmpw(reg1, reg2);
3574 %}
3575
3576 enc_class aarch64_enc_testp(iRegP src) %{
3577 Register reg = as_Register($src$$reg);
3578 __ cmp(reg, zr);
3579 %}
3580
3581 enc_class aarch64_enc_testn(iRegN src) %{
3582 Register reg = as_Register($src$$reg);
3583 __ cmpw(reg, zr);
3584 %}
3585
3586 enc_class aarch64_enc_b(label lbl) %{
3587 Label *L = $lbl$$label;
3588 __ b(*L);
3589 %}
3590
3591 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3592 Label *L = $lbl$$label;
3593 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3594 %}
3595
3596 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3597 Label *L = $lbl$$label;
3598 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3599 %}
3600
3601 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3602 %{
3603 Register sub_reg = as_Register($sub$$reg);
3604 Register super_reg = as_Register($super$$reg);
3605 Register temp_reg = as_Register($temp$$reg);
3606 Register result_reg = as_Register($result$$reg);
3607
3608 Label miss;
3609 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3610 nullptr, &miss,
3611 /*set_cond_codes:*/ true);
3612 if ($primary) {
3613 __ mov(result_reg, zr);
3614 }
3615 __ bind(miss);
3616 %}
3617
3618 enc_class aarch64_enc_java_static_call(method meth) %{
3619 address addr = (address)$meth$$method;
3620 address call;
3621 if (!_method) {
3622 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3623 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3624 if (call == nullptr) {
3625 ciEnv::current()->record_failure("CodeCache is full");
3626 return;
3627 }
3628 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3629 // The NOP here is purely to ensure that eliding a call to
3630 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3631 __ nop();
3632 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3633 } else {
3634 int method_index = resolved_method_index(masm);
3635 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3636 : static_call_Relocation::spec(method_index);
3637 call = __ trampoline_call(Address(addr, rspec));
3638 if (call == nullptr) {
3639 ciEnv::current()->record_failure("CodeCache is full");
3640 return;
3641 }
3642 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3643 // Calls of the same statically bound method can share
3644 // a stub to the interpreter.
3645 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3646 } else {
3647 // Emit stub for static call
3648 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3649 if (stub == nullptr) {
3650 ciEnv::current()->record_failure("CodeCache is full");
3651 return;
3652 }
3653 }
3654 }
3655
3656 __ post_call_nop();
3657
3658 // Only non uncommon_trap calls need to reinitialize ptrue.
3659 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3660 __ reinitialize_ptrue();
3661 }
3662 %}
3663
3664 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3665 int method_index = resolved_method_index(masm);
3666 address call = __ ic_call((address)$meth$$method, method_index);
3667 if (call == nullptr) {
3668 ciEnv::current()->record_failure("CodeCache is full");
3669 return;
3670 }
3671 __ post_call_nop();
3672 if (Compile::current()->max_vector_size() > 0) {
3673 __ reinitialize_ptrue();
3674 }
3675 %}
3676
3677 enc_class aarch64_enc_call_epilog() %{
3678 if (VerifyStackAtCalls) {
3679 // Check that stack depth is unchanged: find majik cookie on stack
3680 __ call_Unimplemented();
3681 }
3682 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3683 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3684 // Search for the corresponding projection, get the register and emit code that initializes it.
3685 uint con = (tf()->range_cc()->cnt() - 1);
3686 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3687 ProjNode* proj = fast_out(i)->as_Proj();
3688 if (proj->_con == con) {
3689 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3690 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3691 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3692 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3693 __ cmp(r0, zr);
3694 __ cset(toReg, Assembler::NE);
3695 if (reg->is_stack()) {
3696 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3697 __ str(toReg, Address(sp, st_off));
3698 }
3699 break;
3700 }
3701 }
3702 if (return_value_is_used()) {
3703 // An inline type is returned as fields in multiple registers.
3704 // R0 either contains an oop if the inline type is buffered or a pointer
3705 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3706 // if the lowest bit is set to allow C2 to use the oop after null checking.
3707 // r0 &= (r0 & 1) - 1
3708 __ andr(rscratch1, r0, 0x1);
3709 __ sub(rscratch1, rscratch1, 0x1);
3710 __ andr(r0, r0, rscratch1);
3711 }
3712 }
3713 %}
3714
3715 enc_class aarch64_enc_java_to_runtime(method meth) %{
3716 // some calls to generated routines (arraycopy code) are scheduled
3717 // by C2 as runtime calls. if so we can call them using a br (they
3718 // will be in a reachable segment) otherwise we have to use a blr
3719 // which loads the absolute address into a register.
3720 address entry = (address)$meth$$method;
3721 CodeBlob *cb = CodeCache::find_blob(entry);
3722 if (cb) {
3723 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3724 if (call == nullptr) {
3725 ciEnv::current()->record_failure("CodeCache is full");
3726 return;
3727 }
3728 __ post_call_nop();
3729 } else {
3730 Label retaddr;
3731 // Make the anchor frame walkable
3732 __ adr(rscratch2, retaddr);
3733 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3734 __ lea(rscratch1, RuntimeAddress(entry));
3735 __ blr(rscratch1);
3736 __ bind(retaddr);
3737 __ post_call_nop();
3738 }
3739 if (Compile::current()->max_vector_size() > 0) {
3740 __ reinitialize_ptrue();
3741 }
3742 %}
3743
3744 enc_class aarch64_enc_rethrow() %{
3745 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3746 %}
3747
3748 enc_class aarch64_enc_ret() %{
3749 #ifdef ASSERT
3750 if (Compile::current()->max_vector_size() > 0) {
3751 __ verify_ptrue();
3752 }
3753 #endif
3754 __ ret(lr);
3755 %}
3756
3757 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3758 Register target_reg = as_Register($jump_target$$reg);
3759 __ br(target_reg);
3760 %}
3761
3762 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3763 Register target_reg = as_Register($jump_target$$reg);
3764 // exception oop should be in r0
3765 // ret addr has been popped into lr
3766 // callee expects it in r3
3767 __ mov(r3, lr);
3768 __ br(target_reg);
3769 %}
3770
3771 %}
3772
3773 //----------FRAME--------------------------------------------------------------
3774 // Definition of frame structure and management information.
3775 //
3776 // S T A C K L A Y O U T Allocators stack-slot number
3777 // | (to get allocators register number
3778 // G Owned by | | v add OptoReg::stack0())
3779 // r CALLER | |
3780 // o | +--------+ pad to even-align allocators stack-slot
3781 // w V | pad0 | numbers; owned by CALLER
3782 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3783 // h ^ | in | 5
3784 // | | args | 4 Holes in incoming args owned by SELF
3785 // | | | | 3
3786 // | | +--------+
3787 // V | | old out| Empty on Intel, window on Sparc
3788 // | old |preserve| Must be even aligned.
3789 // | SP-+--------+----> Matcher::_old_SP, even aligned
3790 // | | in | 3 area for Intel ret address
3791 // Owned by |preserve| Empty on Sparc.
3792 // SELF +--------+
3793 // | | pad2 | 2 pad to align old SP
3794 // | +--------+ 1
3795 // | | locks | 0
3796 // | +--------+----> OptoReg::stack0(), even aligned
3797 // | | pad1 | 11 pad to align new SP
3798 // | +--------+
3799 // | | | 10
3800 // | | spills | 9 spills
3801 // V | | 8 (pad0 slot for callee)
3802 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3803 // ^ | out | 7
3804 // | | args | 6 Holes in outgoing args owned by CALLEE
3805 // Owned by +--------+
3806 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3807 // | new |preserve| Must be even-aligned.
3808 // | SP-+--------+----> Matcher::_new_SP, even aligned
3809 // | | |
3810 //
3811 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3812 // known from SELF's arguments and the Java calling convention.
3813 // Region 6-7 is determined per call site.
3814 // Note 2: If the calling convention leaves holes in the incoming argument
3815 // area, those holes are owned by SELF. Holes in the outgoing area
3816 // are owned by the CALLEE. Holes should not be necessary in the
3817 // incoming area, as the Java calling convention is completely under
3818 // the control of the AD file. Doubles can be sorted and packed to
3819 // avoid holes. Holes in the outgoing arguments may be necessary for
3820 // varargs C calling conventions.
3821 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3822 // even aligned with pad0 as needed.
3823 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3824 // (the latter is true on Intel but is it false on AArch64?)
3825 // region 6-11 is even aligned; it may be padded out more so that
3826 // the region from SP to FP meets the minimum stack alignment.
3827 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3828 // alignment. Region 11, pad1, may be dynamically extended so that
3829 // SP meets the minimum alignment.
3830
3831 frame %{
3832 // These three registers define part of the calling convention
3833 // between compiled code and the interpreter.
3834
3835 // Inline Cache Register or Method for I2C.
3836 inline_cache_reg(R12);
3837
3838 // Number of stack slots consumed by locking an object
3839 sync_stack_slots(2);
3840
3841 // Compiled code's Frame Pointer
3842 frame_pointer(R31);
3843
3844 // Stack alignment requirement
3845 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3846
3847 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3848 // for calls to C. Supports the var-args backing area for register parms.
3849 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3850
3851 // The after-PROLOG location of the return address. Location of
3852 // return address specifies a type (REG or STACK) and a number
3853 // representing the register number (i.e. - use a register name) or
3854 // stack slot.
3855 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3856 // Otherwise, it is above the locks and verification slot and alignment word
3857 // TODO this may well be correct but need to check why that - 2 is there
3858 // ppc port uses 0 but we definitely need to allow for fixed_slots
3859 // which folds in the space used for monitors
3860 return_addr(STACK - 2 +
3861 align_up((Compile::current()->in_preserve_stack_slots() +
3862 Compile::current()->fixed_slots()),
3863 stack_alignment_in_slots()));
3864
3865 // Location of compiled Java return values. Same as C for now.
3866 return_value
3867 %{
3868 // TODO do we allow ideal_reg == Op_RegN???
3869 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3870 "only return normal values");
3871
3872 static const int lo[Op_RegL + 1] = { // enum name
3873 0, // Op_Node
3874 0, // Op_Set
3875 R0_num, // Op_RegN
3876 R0_num, // Op_RegI
3877 R0_num, // Op_RegP
3878 V0_num, // Op_RegF
3879 V0_num, // Op_RegD
3880 R0_num // Op_RegL
3881 };
3882
3883 static const int hi[Op_RegL + 1] = { // enum name
3884 0, // Op_Node
3885 0, // Op_Set
3886 OptoReg::Bad, // Op_RegN
3887 OptoReg::Bad, // Op_RegI
3888 R0_H_num, // Op_RegP
3889 OptoReg::Bad, // Op_RegF
3890 V0_H_num, // Op_RegD
3891 R0_H_num // Op_RegL
3892 };
3893
3894 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3895 %}
3896 %}
3897
3898 //----------ATTRIBUTES---------------------------------------------------------
3899 //----------Operand Attributes-------------------------------------------------
3900 op_attrib op_cost(1); // Required cost attribute
3901
3902 //----------Instruction Attributes---------------------------------------------
3903 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3904 ins_attrib ins_size(32); // Required size attribute (in bits)
3905 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3906 // a non-matching short branch variant
3907 // of some long branch?
3908 ins_attrib ins_alignment(4); // Required alignment attribute (must
3909 // be a power of 2) specifies the
3910 // alignment that some part of the
3911 // instruction (not necessarily the
3912 // start) requires. If > 1, a
3913 // compute_padding() function must be
3914 // provided for the instruction
3915
3916 // Whether this node is expanded during code emission into a sequence of
3917 // instructions and the first instruction can perform an implicit null check.
3918 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3919
3920 //----------OPERANDS-----------------------------------------------------------
3921 // Operand definitions must precede instruction definitions for correct parsing
3922 // in the ADLC because operands constitute user defined types which are used in
3923 // instruction definitions.
3924
3925 //----------Simple Operands----------------------------------------------------
3926
3927 // Integer operands 32 bit
3928 // 32 bit immediate
3929 operand immI()
3930 %{
3931 match(ConI);
3932
3933 op_cost(0);
3934 format %{ %}
3935 interface(CONST_INTER);
3936 %}
3937
3938 // 32 bit zero
3939 operand immI0()
3940 %{
3941 predicate(n->get_int() == 0);
3942 match(ConI);
3943
3944 op_cost(0);
3945 format %{ %}
3946 interface(CONST_INTER);
3947 %}
3948
3949 // 32 bit unit increment
3950 operand immI_1()
3951 %{
3952 predicate(n->get_int() == 1);
3953 match(ConI);
3954
3955 op_cost(0);
3956 format %{ %}
3957 interface(CONST_INTER);
3958 %}
3959
3960 // 32 bit unit decrement
3961 operand immI_M1()
3962 %{
3963 predicate(n->get_int() == -1);
3964 match(ConI);
3965
3966 op_cost(0);
3967 format %{ %}
3968 interface(CONST_INTER);
3969 %}
3970
3971 // Shift values for add/sub extension shift
3972 operand immIExt()
3973 %{
3974 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3975 match(ConI);
3976
3977 op_cost(0);
3978 format %{ %}
3979 interface(CONST_INTER);
3980 %}
3981
3982 operand immI_gt_1()
3983 %{
3984 predicate(n->get_int() > 1);
3985 match(ConI);
3986
3987 op_cost(0);
3988 format %{ %}
3989 interface(CONST_INTER);
3990 %}
3991
3992 operand immI_le_4()
3993 %{
3994 predicate(n->get_int() <= 4);
3995 match(ConI);
3996
3997 op_cost(0);
3998 format %{ %}
3999 interface(CONST_INTER);
4000 %}
4001
4002 operand immI_4()
4003 %{
4004 predicate(n->get_int() == 4);
4005 match(ConI);
4006
4007 op_cost(0);
4008 format %{ %}
4009 interface(CONST_INTER);
4010 %}
4011
4012 operand immI_16()
4013 %{
4014 predicate(n->get_int() == 16);
4015 match(ConI);
4016
4017 op_cost(0);
4018 format %{ %}
4019 interface(CONST_INTER);
4020 %}
4021
4022 operand immI_24()
4023 %{
4024 predicate(n->get_int() == 24);
4025 match(ConI);
4026
4027 op_cost(0);
4028 format %{ %}
4029 interface(CONST_INTER);
4030 %}
4031
4032 operand immI_32()
4033 %{
4034 predicate(n->get_int() == 32);
4035 match(ConI);
4036
4037 op_cost(0);
4038 format %{ %}
4039 interface(CONST_INTER);
4040 %}
4041
4042 operand immI_48()
4043 %{
4044 predicate(n->get_int() == 48);
4045 match(ConI);
4046
4047 op_cost(0);
4048 format %{ %}
4049 interface(CONST_INTER);
4050 %}
4051
4052 operand immI_56()
4053 %{
4054 predicate(n->get_int() == 56);
4055 match(ConI);
4056
4057 op_cost(0);
4058 format %{ %}
4059 interface(CONST_INTER);
4060 %}
4061
4062 operand immI_255()
4063 %{
4064 predicate(n->get_int() == 255);
4065 match(ConI);
4066
4067 op_cost(0);
4068 format %{ %}
4069 interface(CONST_INTER);
4070 %}
4071
4072 operand immI_65535()
4073 %{
4074 predicate(n->get_int() == 65535);
4075 match(ConI);
4076
4077 op_cost(0);
4078 format %{ %}
4079 interface(CONST_INTER);
4080 %}
4081
4082 operand immI_positive()
4083 %{
4084 predicate(n->get_int() > 0);
4085 match(ConI);
4086
4087 op_cost(0);
4088 format %{ %}
4089 interface(CONST_INTER);
4090 %}
4091
4092 // BoolTest condition for signed compare
4093 operand immI_cmp_cond()
4094 %{
4095 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4096 match(ConI);
4097
4098 op_cost(0);
4099 format %{ %}
4100 interface(CONST_INTER);
4101 %}
4102
4103 // BoolTest condition for unsigned compare
4104 operand immI_cmpU_cond()
4105 %{
4106 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4107 match(ConI);
4108
4109 op_cost(0);
4110 format %{ %}
4111 interface(CONST_INTER);
4112 %}
4113
4114 operand immL_255()
4115 %{
4116 predicate(n->get_long() == 255L);
4117 match(ConL);
4118
4119 op_cost(0);
4120 format %{ %}
4121 interface(CONST_INTER);
4122 %}
4123
4124 operand immL_65535()
4125 %{
4126 predicate(n->get_long() == 65535L);
4127 match(ConL);
4128
4129 op_cost(0);
4130 format %{ %}
4131 interface(CONST_INTER);
4132 %}
4133
4134 operand immL_4294967295()
4135 %{
4136 predicate(n->get_long() == 4294967295L);
4137 match(ConL);
4138
4139 op_cost(0);
4140 format %{ %}
4141 interface(CONST_INTER);
4142 %}
4143
4144 operand immL_bitmask()
4145 %{
4146 predicate((n->get_long() != 0)
4147 && ((n->get_long() & 0xc000000000000000l) == 0)
4148 && is_power_of_2(n->get_long() + 1));
4149 match(ConL);
4150
4151 op_cost(0);
4152 format %{ %}
4153 interface(CONST_INTER);
4154 %}
4155
4156 operand immI_bitmask()
4157 %{
4158 predicate((n->get_int() != 0)
4159 && ((n->get_int() & 0xc0000000) == 0)
4160 && is_power_of_2(n->get_int() + 1));
4161 match(ConI);
4162
4163 op_cost(0);
4164 format %{ %}
4165 interface(CONST_INTER);
4166 %}
4167
4168 operand immL_positive_bitmaskI()
4169 %{
4170 predicate((n->get_long() != 0)
4171 && ((julong)n->get_long() < 0x80000000ULL)
4172 && is_power_of_2(n->get_long() + 1));
4173 match(ConL);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 // Scale values for scaled offset addressing modes (up to long but not quad)
4181 operand immIScale()
4182 %{
4183 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4184 match(ConI);
4185
4186 op_cost(0);
4187 format %{ %}
4188 interface(CONST_INTER);
4189 %}
4190
4191 // 5 bit signed integer
4192 operand immI5()
4193 %{
4194 predicate(Assembler::is_simm(n->get_int(), 5));
4195 match(ConI);
4196
4197 op_cost(0);
4198 format %{ %}
4199 interface(CONST_INTER);
4200 %}
4201
4202 // 7 bit unsigned integer
4203 operand immIU7()
4204 %{
4205 predicate(Assembler::is_uimm(n->get_int(), 7));
4206 match(ConI);
4207
4208 op_cost(0);
4209 format %{ %}
4210 interface(CONST_INTER);
4211 %}
4212
4213 // Offset for scaled or unscaled immediate loads and stores
4214 operand immIOffset()
4215 %{
4216 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4217 match(ConI);
4218
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 operand immIOffset1()
4225 %{
4226 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4227 match(ConI);
4228
4229 op_cost(0);
4230 format %{ %}
4231 interface(CONST_INTER);
4232 %}
4233
4234 operand immIOffset2()
4235 %{
4236 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4237 match(ConI);
4238
4239 op_cost(0);
4240 format %{ %}
4241 interface(CONST_INTER);
4242 %}
4243
4244 operand immIOffset4()
4245 %{
4246 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4247 match(ConI);
4248
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 operand immIOffset8()
4255 %{
4256 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4257 match(ConI);
4258
4259 op_cost(0);
4260 format %{ %}
4261 interface(CONST_INTER);
4262 %}
4263
4264 operand immIOffset16()
4265 %{
4266 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4267 match(ConI);
4268
4269 op_cost(0);
4270 format %{ %}
4271 interface(CONST_INTER);
4272 %}
4273
4274 operand immLOffset()
4275 %{
4276 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4277 match(ConL);
4278
4279 op_cost(0);
4280 format %{ %}
4281 interface(CONST_INTER);
4282 %}
4283
4284 operand immLoffset1()
4285 %{
4286 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4287 match(ConL);
4288
4289 op_cost(0);
4290 format %{ %}
4291 interface(CONST_INTER);
4292 %}
4293
4294 operand immLoffset2()
4295 %{
4296 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4297 match(ConL);
4298
4299 op_cost(0);
4300 format %{ %}
4301 interface(CONST_INTER);
4302 %}
4303
4304 operand immLoffset4()
4305 %{
4306 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4307 match(ConL);
4308
4309 op_cost(0);
4310 format %{ %}
4311 interface(CONST_INTER);
4312 %}
4313
4314 operand immLoffset8()
4315 %{
4316 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4317 match(ConL);
4318
4319 op_cost(0);
4320 format %{ %}
4321 interface(CONST_INTER);
4322 %}
4323
4324 operand immLoffset16()
4325 %{
4326 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4327 match(ConL);
4328
4329 op_cost(0);
4330 format %{ %}
4331 interface(CONST_INTER);
4332 %}
4333
4334 // 5 bit signed long integer
4335 operand immL5()
4336 %{
4337 predicate(Assembler::is_simm(n->get_long(), 5));
4338 match(ConL);
4339
4340 op_cost(0);
4341 format %{ %}
4342 interface(CONST_INTER);
4343 %}
4344
4345 // 7 bit unsigned long integer
4346 operand immLU7()
4347 %{
4348 predicate(Assembler::is_uimm(n->get_long(), 7));
4349 match(ConL);
4350
4351 op_cost(0);
4352 format %{ %}
4353 interface(CONST_INTER);
4354 %}
4355
4356 // 8 bit signed value.
4357 operand immI8()
4358 %{
4359 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4360 match(ConI);
4361
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 // 8 bit signed value (simm8), or #simm8 LSL 8.
4368 operand immIDupV()
4369 %{
4370 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4371 match(ConI);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 // 8 bit signed value (simm8), or #simm8 LSL 8.
4379 operand immLDupV()
4380 %{
4381 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4382 match(ConL);
4383
4384 op_cost(0);
4385 format %{ %}
4386 interface(CONST_INTER);
4387 %}
4388
4389 // 8 bit signed value (simm8), or #simm8 LSL 8.
4390 operand immHDupV()
4391 %{
4392 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4393 match(ConH);
4394
4395 op_cost(0);
4396 format %{ %}
4397 interface(CONST_INTER);
4398 %}
4399
4400 // 8 bit integer valid for vector add sub immediate
4401 operand immBAddSubV()
4402 %{
4403 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4404 match(ConI);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 // 32 bit integer valid for add sub immediate
4412 operand immIAddSub()
4413 %{
4414 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4415 match(ConI);
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 // 32 bit integer valid for vector add sub immediate
4422 operand immIAddSubV()
4423 %{
4424 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4425 match(ConI);
4426
4427 op_cost(0);
4428 format %{ %}
4429 interface(CONST_INTER);
4430 %}
4431
4432 // 32 bit unsigned integer valid for logical immediate
4433
4434 operand immBLog()
4435 %{
4436 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4437 match(ConI);
4438
4439 op_cost(0);
4440 format %{ %}
4441 interface(CONST_INTER);
4442 %}
4443
4444 operand immSLog()
4445 %{
4446 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4447 match(ConI);
4448
4449 op_cost(0);
4450 format %{ %}
4451 interface(CONST_INTER);
4452 %}
4453
4454 operand immILog()
4455 %{
4456 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4457 match(ConI);
4458
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 // Integer operands 64 bit
4465 // 64 bit immediate
4466 operand immL()
4467 %{
4468 match(ConL);
4469
4470 op_cost(0);
4471 format %{ %}
4472 interface(CONST_INTER);
4473 %}
4474
4475 // 64 bit zero
4476 operand immL0()
4477 %{
4478 predicate(n->get_long() == 0);
4479 match(ConL);
4480
4481 op_cost(0);
4482 format %{ %}
4483 interface(CONST_INTER);
4484 %}
4485
4486 // 64 bit unit decrement
4487 operand immL_M1()
4488 %{
4489 predicate(n->get_long() == -1);
4490 match(ConL);
4491
4492 op_cost(0);
4493 format %{ %}
4494 interface(CONST_INTER);
4495 %}
4496
4497 // 64 bit integer valid for add sub immediate
4498 operand immLAddSub()
4499 %{
4500 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4501 match(ConL);
4502 op_cost(0);
4503 format %{ %}
4504 interface(CONST_INTER);
4505 %}
4506
4507 // 64 bit integer valid for addv subv immediate
4508 operand immLAddSubV()
4509 %{
4510 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4511 match(ConL);
4512
4513 op_cost(0);
4514 format %{ %}
4515 interface(CONST_INTER);
4516 %}
4517
4518 // 64 bit integer valid for logical immediate
4519 operand immLLog()
4520 %{
4521 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4522 match(ConL);
4523 op_cost(0);
4524 format %{ %}
4525 interface(CONST_INTER);
4526 %}
4527
4528 // Long Immediate: low 32-bit mask
4529 operand immL_32bits()
4530 %{
4531 predicate(n->get_long() == 0xFFFFFFFFL);
4532 match(ConL);
4533 op_cost(0);
4534 format %{ %}
4535 interface(CONST_INTER);
4536 %}
4537
4538 // Pointer operands
4539 // Pointer Immediate
4540 operand immP()
4541 %{
4542 match(ConP);
4543
4544 op_cost(0);
4545 format %{ %}
4546 interface(CONST_INTER);
4547 %}
4548
4549 // nullptr Pointer Immediate
4550 operand immP0()
4551 %{
4552 predicate(n->get_ptr() == 0);
4553 match(ConP);
4554
4555 op_cost(0);
4556 format %{ %}
4557 interface(CONST_INTER);
4558 %}
4559
4560 // Pointer Immediate One
4561 // this is used in object initialization (initial object header)
4562 operand immP_1()
4563 %{
4564 predicate(n->get_ptr() == 1);
4565 match(ConP);
4566
4567 op_cost(0);
4568 format %{ %}
4569 interface(CONST_INTER);
4570 %}
4571
4572 // AOT Runtime Constants Address
4573 operand immAOTRuntimeConstantsAddress()
4574 %{
4575 // Check if the address is in the range of AOT Runtime Constants
4576 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4577 match(ConP);
4578
4579 op_cost(0);
4580 format %{ %}
4581 interface(CONST_INTER);
4582 %}
4583
4584 // Float and Double operands
4585 // Double Immediate
4586 operand immD()
4587 %{
4588 match(ConD);
4589 op_cost(0);
4590 format %{ %}
4591 interface(CONST_INTER);
4592 %}
4593
4594 // Double Immediate: +0.0d
4595 operand immD0()
4596 %{
4597 predicate(jlong_cast(n->getd()) == 0);
4598 match(ConD);
4599
4600 op_cost(0);
4601 format %{ %}
4602 interface(CONST_INTER);
4603 %}
4604
4605 // constant 'double +0.0'.
4606 operand immDPacked()
4607 %{
4608 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4609 match(ConD);
4610 op_cost(0);
4611 format %{ %}
4612 interface(CONST_INTER);
4613 %}
4614
4615 // Float Immediate
4616 operand immF()
4617 %{
4618 match(ConF);
4619 op_cost(0);
4620 format %{ %}
4621 interface(CONST_INTER);
4622 %}
4623
4624 // Float Immediate: +0.0f.
4625 operand immF0()
4626 %{
4627 predicate(jint_cast(n->getf()) == 0);
4628 match(ConF);
4629
4630 op_cost(0);
4631 format %{ %}
4632 interface(CONST_INTER);
4633 %}
4634
4635 // Half Float (FP16) Immediate
4636 operand immH()
4637 %{
4638 match(ConH);
4639 op_cost(0);
4640 format %{ %}
4641 interface(CONST_INTER);
4642 %}
4643
4644 //
4645 operand immFPacked()
4646 %{
4647 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4648 match(ConF);
4649 op_cost(0);
4650 format %{ %}
4651 interface(CONST_INTER);
4652 %}
4653
4654 // Narrow pointer operands
4655 // Narrow Pointer Immediate
4656 operand immN()
4657 %{
4658 match(ConN);
4659
4660 op_cost(0);
4661 format %{ %}
4662 interface(CONST_INTER);
4663 %}
4664
4665 // Narrow nullptr Pointer Immediate
4666 operand immN0()
4667 %{
4668 predicate(n->get_narrowcon() == 0);
4669 match(ConN);
4670
4671 op_cost(0);
4672 format %{ %}
4673 interface(CONST_INTER);
4674 %}
4675
4676 operand immNKlass()
4677 %{
4678 match(ConNKlass);
4679
4680 op_cost(0);
4681 format %{ %}
4682 interface(CONST_INTER);
4683 %}
4684
4685 // Integer 32 bit Register Operands
4686 // Integer 32 bitRegister (excludes SP)
4687 operand iRegI()
4688 %{
4689 constraint(ALLOC_IN_RC(any_reg32));
4690 match(RegI);
4691 match(iRegINoSp);
4692 op_cost(0);
4693 format %{ %}
4694 interface(REG_INTER);
4695 %}
4696
4697 // Integer 32 bit Register not Special
4698 operand iRegINoSp()
4699 %{
4700 constraint(ALLOC_IN_RC(no_special_reg32));
4701 match(RegI);
4702 op_cost(0);
4703 format %{ %}
4704 interface(REG_INTER);
4705 %}
4706
4707 // Integer 64 bit Register Operands
4708 // Integer 64 bit Register (includes SP)
4709 operand iRegL()
4710 %{
4711 constraint(ALLOC_IN_RC(any_reg));
4712 match(RegL);
4713 match(iRegLNoSp);
4714 op_cost(0);
4715 format %{ %}
4716 interface(REG_INTER);
4717 %}
4718
4719 // Integer 64 bit Register not Special
4720 operand iRegLNoSp()
4721 %{
4722 constraint(ALLOC_IN_RC(no_special_reg));
4723 match(RegL);
4724 match(iRegL_R0);
4725 format %{ %}
4726 interface(REG_INTER);
4727 %}
4728
4729 // Pointer Register Operands
4730 // Pointer Register
4731 operand iRegP()
4732 %{
4733 constraint(ALLOC_IN_RC(ptr_reg));
4734 match(RegP);
4735 match(iRegPNoSp);
4736 match(iRegP_R0);
4737 //match(iRegP_R2);
4738 //match(iRegP_R4);
4739 match(iRegP_R5);
4740 match(thread_RegP);
4741 op_cost(0);
4742 format %{ %}
4743 interface(REG_INTER);
4744 %}
4745
4746 // Pointer 64 bit Register not Special
4747 operand iRegPNoSp()
4748 %{
4749 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4750 match(RegP);
4751 // match(iRegP);
4752 // match(iRegP_R0);
4753 // match(iRegP_R2);
4754 // match(iRegP_R4);
4755 // match(iRegP_R5);
4756 // match(thread_RegP);
4757 op_cost(0);
4758 format %{ %}
4759 interface(REG_INTER);
4760 %}
4761
4762 // This operand is not allowed to use rfp even if
4763 // rfp is not used to hold the frame pointer.
4764 operand iRegPNoSpNoRfp()
4765 %{
4766 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4767 match(RegP);
4768 match(iRegPNoSp);
4769 op_cost(0);
4770 format %{ %}
4771 interface(REG_INTER);
4772 %}
4773
4774 // Pointer 64 bit Register R0 only
4775 operand iRegP_R0()
4776 %{
4777 constraint(ALLOC_IN_RC(r0_reg));
4778 match(RegP);
4779 // match(iRegP);
4780 match(iRegPNoSp);
4781 op_cost(0);
4782 format %{ %}
4783 interface(REG_INTER);
4784 %}
4785
4786 // Pointer 64 bit Register R1 only
4787 operand iRegP_R1()
4788 %{
4789 constraint(ALLOC_IN_RC(r1_reg));
4790 match(RegP);
4791 // match(iRegP);
4792 match(iRegPNoSp);
4793 op_cost(0);
4794 format %{ %}
4795 interface(REG_INTER);
4796 %}
4797
4798 // Pointer 64 bit Register R2 only
4799 operand iRegP_R2()
4800 %{
4801 constraint(ALLOC_IN_RC(r2_reg));
4802 match(RegP);
4803 // match(iRegP);
4804 match(iRegPNoSp);
4805 op_cost(0);
4806 format %{ %}
4807 interface(REG_INTER);
4808 %}
4809
4810 // Pointer 64 bit Register R3 only
4811 operand iRegP_R3()
4812 %{
4813 constraint(ALLOC_IN_RC(r3_reg));
4814 match(RegP);
4815 // match(iRegP);
4816 match(iRegPNoSp);
4817 op_cost(0);
4818 format %{ %}
4819 interface(REG_INTER);
4820 %}
4821
4822 // Pointer 64 bit Register R4 only
4823 operand iRegP_R4()
4824 %{
4825 constraint(ALLOC_IN_RC(r4_reg));
4826 match(RegP);
4827 // match(iRegP);
4828 match(iRegPNoSp);
4829 op_cost(0);
4830 format %{ %}
4831 interface(REG_INTER);
4832 %}
4833
4834 // Pointer 64 bit Register R5 only
4835 operand iRegP_R5()
4836 %{
4837 constraint(ALLOC_IN_RC(r5_reg));
4838 match(RegP);
4839 // match(iRegP);
4840 match(iRegPNoSp);
4841 op_cost(0);
4842 format %{ %}
4843 interface(REG_INTER);
4844 %}
4845
4846 // Pointer 64 bit Register R10 only
4847 operand iRegP_R10()
4848 %{
4849 constraint(ALLOC_IN_RC(r10_reg));
4850 match(RegP);
4851 // match(iRegP);
4852 match(iRegPNoSp);
4853 op_cost(0);
4854 format %{ %}
4855 interface(REG_INTER);
4856 %}
4857
4858 // Long 64 bit Register R0 only
4859 operand iRegL_R0()
4860 %{
4861 constraint(ALLOC_IN_RC(r0_reg));
4862 match(RegL);
4863 match(iRegLNoSp);
4864 op_cost(0);
4865 format %{ %}
4866 interface(REG_INTER);
4867 %}
4868
4869 // Long 64 bit Register R11 only
4870 operand iRegL_R11()
4871 %{
4872 constraint(ALLOC_IN_RC(r11_reg));
4873 match(RegL);
4874 match(iRegLNoSp);
4875 op_cost(0);
4876 format %{ %}
4877 interface(REG_INTER);
4878 %}
4879
4880 // Register R0 only
4881 operand iRegI_R0()
4882 %{
4883 constraint(ALLOC_IN_RC(int_r0_reg));
4884 match(RegI);
4885 match(iRegINoSp);
4886 op_cost(0);
4887 format %{ %}
4888 interface(REG_INTER);
4889 %}
4890
4891 // Register R2 only
4892 operand iRegI_R2()
4893 %{
4894 constraint(ALLOC_IN_RC(int_r2_reg));
4895 match(RegI);
4896 match(iRegINoSp);
4897 op_cost(0);
4898 format %{ %}
4899 interface(REG_INTER);
4900 %}
4901
4902 // Register R3 only
4903 operand iRegI_R3()
4904 %{
4905 constraint(ALLOC_IN_RC(int_r3_reg));
4906 match(RegI);
4907 match(iRegINoSp);
4908 op_cost(0);
4909 format %{ %}
4910 interface(REG_INTER);
4911 %}
4912
4913
4914 // Register R4 only
4915 operand iRegI_R4()
4916 %{
4917 constraint(ALLOC_IN_RC(int_r4_reg));
4918 match(RegI);
4919 match(iRegINoSp);
4920 op_cost(0);
4921 format %{ %}
4922 interface(REG_INTER);
4923 %}
4924
4925
4926 // Pointer Register Operands
4927 // Narrow Pointer Register
4928 operand iRegN()
4929 %{
4930 constraint(ALLOC_IN_RC(any_reg32));
4931 match(RegN);
4932 match(iRegNNoSp);
4933 op_cost(0);
4934 format %{ %}
4935 interface(REG_INTER);
4936 %}
4937
4938 // Integer 64 bit Register not Special
4939 operand iRegNNoSp()
4940 %{
4941 constraint(ALLOC_IN_RC(no_special_reg32));
4942 match(RegN);
4943 op_cost(0);
4944 format %{ %}
4945 interface(REG_INTER);
4946 %}
4947
4948 // Float Register
4949 // Float register operands
4950 operand vRegF()
4951 %{
4952 constraint(ALLOC_IN_RC(float_reg));
4953 match(RegF);
4954
4955 op_cost(0);
4956 format %{ %}
4957 interface(REG_INTER);
4958 %}
4959
4960 // Double Register
4961 // Double register operands
4962 operand vRegD()
4963 %{
4964 constraint(ALLOC_IN_RC(double_reg));
4965 match(RegD);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 // Generic vector class. This will be used for
4973 // all vector operands, including NEON and SVE.
4974 operand vReg()
4975 %{
4976 constraint(ALLOC_IN_RC(dynamic));
4977 match(VecA);
4978 match(VecD);
4979 match(VecX);
4980
4981 op_cost(0);
4982 format %{ %}
4983 interface(REG_INTER);
4984 %}
4985
4986 operand vReg_V10()
4987 %{
4988 constraint(ALLOC_IN_RC(v10_veca_reg));
4989 match(vReg);
4990
4991 op_cost(0);
4992 format %{ %}
4993 interface(REG_INTER);
4994 %}
4995
4996 operand vReg_V11()
4997 %{
4998 constraint(ALLOC_IN_RC(v11_veca_reg));
4999 match(vReg);
5000
5001 op_cost(0);
5002 format %{ %}
5003 interface(REG_INTER);
5004 %}
5005
5006 operand vReg_V12()
5007 %{
5008 constraint(ALLOC_IN_RC(v12_veca_reg));
5009 match(vReg);
5010
5011 op_cost(0);
5012 format %{ %}
5013 interface(REG_INTER);
5014 %}
5015
5016 operand vReg_V13()
5017 %{
5018 constraint(ALLOC_IN_RC(v13_veca_reg));
5019 match(vReg);
5020
5021 op_cost(0);
5022 format %{ %}
5023 interface(REG_INTER);
5024 %}
5025
5026 operand vReg_V17()
5027 %{
5028 constraint(ALLOC_IN_RC(v17_veca_reg));
5029 match(vReg);
5030
5031 op_cost(0);
5032 format %{ %}
5033 interface(REG_INTER);
5034 %}
5035
5036 operand vReg_V18()
5037 %{
5038 constraint(ALLOC_IN_RC(v18_veca_reg));
5039 match(vReg);
5040
5041 op_cost(0);
5042 format %{ %}
5043 interface(REG_INTER);
5044 %}
5045
5046 operand vReg_V23()
5047 %{
5048 constraint(ALLOC_IN_RC(v23_veca_reg));
5049 match(vReg);
5050
5051 op_cost(0);
5052 format %{ %}
5053 interface(REG_INTER);
5054 %}
5055
5056 operand vReg_V24()
5057 %{
5058 constraint(ALLOC_IN_RC(v24_veca_reg));
5059 match(vReg);
5060
5061 op_cost(0);
5062 format %{ %}
5063 interface(REG_INTER);
5064 %}
5065
5066 operand vecA()
5067 %{
5068 constraint(ALLOC_IN_RC(vectora_reg));
5069 match(VecA);
5070
5071 op_cost(0);
5072 format %{ %}
5073 interface(REG_INTER);
5074 %}
5075
5076 operand vecD()
5077 %{
5078 constraint(ALLOC_IN_RC(vectord_reg));
5079 match(VecD);
5080
5081 op_cost(0);
5082 format %{ %}
5083 interface(REG_INTER);
5084 %}
5085
5086 operand vecX()
5087 %{
5088 constraint(ALLOC_IN_RC(vectorx_reg));
5089 match(VecX);
5090
5091 op_cost(0);
5092 format %{ %}
5093 interface(REG_INTER);
5094 %}
5095
5096 operand vRegD_V0()
5097 %{
5098 constraint(ALLOC_IN_RC(v0_reg));
5099 match(RegD);
5100 op_cost(0);
5101 format %{ %}
5102 interface(REG_INTER);
5103 %}
5104
5105 operand vRegD_V1()
5106 %{
5107 constraint(ALLOC_IN_RC(v1_reg));
5108 match(RegD);
5109 op_cost(0);
5110 format %{ %}
5111 interface(REG_INTER);
5112 %}
5113
5114 operand vRegD_V2()
5115 %{
5116 constraint(ALLOC_IN_RC(v2_reg));
5117 match(RegD);
5118 op_cost(0);
5119 format %{ %}
5120 interface(REG_INTER);
5121 %}
5122
5123 operand vRegD_V3()
5124 %{
5125 constraint(ALLOC_IN_RC(v3_reg));
5126 match(RegD);
5127 op_cost(0);
5128 format %{ %}
5129 interface(REG_INTER);
5130 %}
5131
5132 operand vRegD_V4()
5133 %{
5134 constraint(ALLOC_IN_RC(v4_reg));
5135 match(RegD);
5136 op_cost(0);
5137 format %{ %}
5138 interface(REG_INTER);
5139 %}
5140
5141 operand vRegD_V5()
5142 %{
5143 constraint(ALLOC_IN_RC(v5_reg));
5144 match(RegD);
5145 op_cost(0);
5146 format %{ %}
5147 interface(REG_INTER);
5148 %}
5149
5150 operand vRegD_V6()
5151 %{
5152 constraint(ALLOC_IN_RC(v6_reg));
5153 match(RegD);
5154 op_cost(0);
5155 format %{ %}
5156 interface(REG_INTER);
5157 %}
5158
5159 operand vRegD_V7()
5160 %{
5161 constraint(ALLOC_IN_RC(v7_reg));
5162 match(RegD);
5163 op_cost(0);
5164 format %{ %}
5165 interface(REG_INTER);
5166 %}
5167
5168 operand vRegD_V12()
5169 %{
5170 constraint(ALLOC_IN_RC(v12_reg));
5171 match(RegD);
5172 op_cost(0);
5173 format %{ %}
5174 interface(REG_INTER);
5175 %}
5176
5177 operand vRegD_V13()
5178 %{
5179 constraint(ALLOC_IN_RC(v13_reg));
5180 match(RegD);
5181 op_cost(0);
5182 format %{ %}
5183 interface(REG_INTER);
5184 %}
5185
5186 operand pReg()
5187 %{
5188 constraint(ALLOC_IN_RC(pr_reg));
5189 match(RegVectMask);
5190 match(pRegGov);
5191 op_cost(0);
5192 format %{ %}
5193 interface(REG_INTER);
5194 %}
5195
5196 operand pRegGov()
5197 %{
5198 constraint(ALLOC_IN_RC(gov_pr));
5199 match(RegVectMask);
5200 match(pReg);
5201 op_cost(0);
5202 format %{ %}
5203 interface(REG_INTER);
5204 %}
5205
5206 operand pRegGov_P0()
5207 %{
5208 constraint(ALLOC_IN_RC(p0_reg));
5209 match(RegVectMask);
5210 op_cost(0);
5211 format %{ %}
5212 interface(REG_INTER);
5213 %}
5214
5215 operand pRegGov_P1()
5216 %{
5217 constraint(ALLOC_IN_RC(p1_reg));
5218 match(RegVectMask);
5219 op_cost(0);
5220 format %{ %}
5221 interface(REG_INTER);
5222 %}
5223
5224 // Flags register, used as output of signed compare instructions
5225
5226 // note that on AArch64 we also use this register as the output for
5227 // for floating point compare instructions (CmpF CmpD). this ensures
5228 // that ordered inequality tests use GT, GE, LT or LE none of which
5229 // pass through cases where the result is unordered i.e. one or both
5230 // inputs to the compare is a NaN. this means that the ideal code can
5231 // replace e.g. a GT with an LE and not end up capturing the NaN case
5232 // (where the comparison should always fail). EQ and NE tests are
5233 // always generated in ideal code so that unordered folds into the NE
5234 // case, matching the behaviour of AArch64 NE.
5235 //
5236 // This differs from x86 where the outputs of FP compares use a
5237 // special FP flags registers and where compares based on this
5238 // register are distinguished into ordered inequalities (cmpOpUCF) and
5239 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5240 // to explicitly handle the unordered case in branches. x86 also has
5241 // to include extra CMoveX rules to accept a cmpOpUCF input.
5242
5243 operand rFlagsReg()
5244 %{
5245 constraint(ALLOC_IN_RC(int_flags));
5246 match(RegFlags);
5247
5248 op_cost(0);
5249 format %{ "RFLAGS" %}
5250 interface(REG_INTER);
5251 %}
5252
5253 // Flags register, used as output of unsigned compare instructions
5254 operand rFlagsRegU()
5255 %{
5256 constraint(ALLOC_IN_RC(int_flags));
5257 match(RegFlags);
5258
5259 op_cost(0);
5260 format %{ "RFLAGSU" %}
5261 interface(REG_INTER);
5262 %}
5263
5264 // Special Registers
5265
5266 // Method Register
5267 operand inline_cache_RegP(iRegP reg)
5268 %{
5269 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5270 match(reg);
5271 match(iRegPNoSp);
5272 op_cost(0);
5273 format %{ %}
5274 interface(REG_INTER);
5275 %}
5276
5277 // Thread Register
5278 operand thread_RegP(iRegP reg)
5279 %{
5280 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5281 match(reg);
5282 op_cost(0);
5283 format %{ %}
5284 interface(REG_INTER);
5285 %}
5286
5287 //----------Memory Operands----------------------------------------------------
5288
5289 operand indirect(iRegP reg)
5290 %{
5291 constraint(ALLOC_IN_RC(ptr_reg));
5292 match(reg);
5293 op_cost(0);
5294 format %{ "[$reg]" %}
5295 interface(MEMORY_INTER) %{
5296 base($reg);
5297 index(0xffffffff);
5298 scale(0x0);
5299 disp(0x0);
5300 %}
5301 %}
5302
5303 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5304 %{
5305 constraint(ALLOC_IN_RC(ptr_reg));
5306 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5307 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5308 op_cost(0);
5309 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5310 interface(MEMORY_INTER) %{
5311 base($reg);
5312 index($ireg);
5313 scale($scale);
5314 disp(0x0);
5315 %}
5316 %}
5317
5318 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5319 %{
5320 constraint(ALLOC_IN_RC(ptr_reg));
5321 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5322 match(AddP reg (LShiftL lreg scale));
5323 op_cost(0);
5324 format %{ "$reg, $lreg lsl($scale)" %}
5325 interface(MEMORY_INTER) %{
5326 base($reg);
5327 index($lreg);
5328 scale($scale);
5329 disp(0x0);
5330 %}
5331 %}
5332
5333 operand indIndexI2L(iRegP reg, iRegI ireg)
5334 %{
5335 constraint(ALLOC_IN_RC(ptr_reg));
5336 match(AddP reg (ConvI2L ireg));
5337 op_cost(0);
5338 format %{ "$reg, $ireg, 0, I2L" %}
5339 interface(MEMORY_INTER) %{
5340 base($reg);
5341 index($ireg);
5342 scale(0x0);
5343 disp(0x0);
5344 %}
5345 %}
5346
5347 operand indIndex(iRegP reg, iRegL lreg)
5348 %{
5349 constraint(ALLOC_IN_RC(ptr_reg));
5350 match(AddP reg lreg);
5351 op_cost(0);
5352 format %{ "$reg, $lreg" %}
5353 interface(MEMORY_INTER) %{
5354 base($reg);
5355 index($lreg);
5356 scale(0x0);
5357 disp(0x0);
5358 %}
5359 %}
5360
5361 operand indOffI1(iRegP reg, immIOffset1 off)
5362 %{
5363 constraint(ALLOC_IN_RC(ptr_reg));
5364 match(AddP reg off);
5365 op_cost(0);
5366 format %{ "[$reg, $off]" %}
5367 interface(MEMORY_INTER) %{
5368 base($reg);
5369 index(0xffffffff);
5370 scale(0x0);
5371 disp($off);
5372 %}
5373 %}
5374
5375 operand indOffI2(iRegP reg, immIOffset2 off)
5376 %{
5377 constraint(ALLOC_IN_RC(ptr_reg));
5378 match(AddP reg off);
5379 op_cost(0);
5380 format %{ "[$reg, $off]" %}
5381 interface(MEMORY_INTER) %{
5382 base($reg);
5383 index(0xffffffff);
5384 scale(0x0);
5385 disp($off);
5386 %}
5387 %}
5388
5389 operand indOffI4(iRegP reg, immIOffset4 off)
5390 %{
5391 constraint(ALLOC_IN_RC(ptr_reg));
5392 match(AddP reg off);
5393 op_cost(0);
5394 format %{ "[$reg, $off]" %}
5395 interface(MEMORY_INTER) %{
5396 base($reg);
5397 index(0xffffffff);
5398 scale(0x0);
5399 disp($off);
5400 %}
5401 %}
5402
5403 operand indOffI8(iRegP reg, immIOffset8 off)
5404 %{
5405 constraint(ALLOC_IN_RC(ptr_reg));
5406 match(AddP reg off);
5407 op_cost(0);
5408 format %{ "[$reg, $off]" %}
5409 interface(MEMORY_INTER) %{
5410 base($reg);
5411 index(0xffffffff);
5412 scale(0x0);
5413 disp($off);
5414 %}
5415 %}
5416
5417 operand indOffI16(iRegP reg, immIOffset16 off)
5418 %{
5419 constraint(ALLOC_IN_RC(ptr_reg));
5420 match(AddP reg off);
5421 op_cost(0);
5422 format %{ "[$reg, $off]" %}
5423 interface(MEMORY_INTER) %{
5424 base($reg);
5425 index(0xffffffff);
5426 scale(0x0);
5427 disp($off);
5428 %}
5429 %}
5430
5431 operand indOffL1(iRegP reg, immLoffset1 off)
5432 %{
5433 constraint(ALLOC_IN_RC(ptr_reg));
5434 match(AddP reg off);
5435 op_cost(0);
5436 format %{ "[$reg, $off]" %}
5437 interface(MEMORY_INTER) %{
5438 base($reg);
5439 index(0xffffffff);
5440 scale(0x0);
5441 disp($off);
5442 %}
5443 %}
5444
5445 operand indOffL2(iRegP reg, immLoffset2 off)
5446 %{
5447 constraint(ALLOC_IN_RC(ptr_reg));
5448 match(AddP reg off);
5449 op_cost(0);
5450 format %{ "[$reg, $off]" %}
5451 interface(MEMORY_INTER) %{
5452 base($reg);
5453 index(0xffffffff);
5454 scale(0x0);
5455 disp($off);
5456 %}
5457 %}
5458
5459 operand indOffL4(iRegP reg, immLoffset4 off)
5460 %{
5461 constraint(ALLOC_IN_RC(ptr_reg));
5462 match(AddP reg off);
5463 op_cost(0);
5464 format %{ "[$reg, $off]" %}
5465 interface(MEMORY_INTER) %{
5466 base($reg);
5467 index(0xffffffff);
5468 scale(0x0);
5469 disp($off);
5470 %}
5471 %}
5472
5473 operand indOffL8(iRegP reg, immLoffset8 off)
5474 %{
5475 constraint(ALLOC_IN_RC(ptr_reg));
5476 match(AddP reg off);
5477 op_cost(0);
5478 format %{ "[$reg, $off]" %}
5479 interface(MEMORY_INTER) %{
5480 base($reg);
5481 index(0xffffffff);
5482 scale(0x0);
5483 disp($off);
5484 %}
5485 %}
5486
5487 operand indOffL16(iRegP reg, immLoffset16 off)
5488 %{
5489 constraint(ALLOC_IN_RC(ptr_reg));
5490 match(AddP reg off);
5491 op_cost(0);
5492 format %{ "[$reg, $off]" %}
5493 interface(MEMORY_INTER) %{
5494 base($reg);
5495 index(0xffffffff);
5496 scale(0x0);
5497 disp($off);
5498 %}
5499 %}
5500
5501 operand indirectX2P(iRegL reg)
5502 %{
5503 constraint(ALLOC_IN_RC(ptr_reg));
5504 match(CastX2P reg);
5505 op_cost(0);
5506 format %{ "[$reg]\t# long -> ptr" %}
5507 interface(MEMORY_INTER) %{
5508 base($reg);
5509 index(0xffffffff);
5510 scale(0x0);
5511 disp(0x0);
5512 %}
5513 %}
5514
5515 operand indOffX2P(iRegL reg, immLOffset off)
5516 %{
5517 constraint(ALLOC_IN_RC(ptr_reg));
5518 match(AddP (CastX2P reg) off);
5519 op_cost(0);
5520 format %{ "[$reg, $off]\t# long -> ptr" %}
5521 interface(MEMORY_INTER) %{
5522 base($reg);
5523 index(0xffffffff);
5524 scale(0x0);
5525 disp($off);
5526 %}
5527 %}
5528
5529 operand indirectN(iRegN reg)
5530 %{
5531 predicate(CompressedOops::shift() == 0);
5532 constraint(ALLOC_IN_RC(ptr_reg));
5533 match(DecodeN reg);
5534 op_cost(0);
5535 format %{ "[$reg]\t# narrow" %}
5536 interface(MEMORY_INTER) %{
5537 base($reg);
5538 index(0xffffffff);
5539 scale(0x0);
5540 disp(0x0);
5541 %}
5542 %}
5543
5544 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5545 %{
5546 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5547 constraint(ALLOC_IN_RC(ptr_reg));
5548 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5549 op_cost(0);
5550 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5551 interface(MEMORY_INTER) %{
5552 base($reg);
5553 index($ireg);
5554 scale($scale);
5555 disp(0x0);
5556 %}
5557 %}
5558
5559 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5560 %{
5561 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5562 constraint(ALLOC_IN_RC(ptr_reg));
5563 match(AddP (DecodeN reg) (LShiftL lreg scale));
5564 op_cost(0);
5565 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5566 interface(MEMORY_INTER) %{
5567 base($reg);
5568 index($lreg);
5569 scale($scale);
5570 disp(0x0);
5571 %}
5572 %}
5573
5574 operand indIndexI2LN(iRegN reg, iRegI ireg)
5575 %{
5576 predicate(CompressedOops::shift() == 0);
5577 constraint(ALLOC_IN_RC(ptr_reg));
5578 match(AddP (DecodeN reg) (ConvI2L ireg));
5579 op_cost(0);
5580 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5581 interface(MEMORY_INTER) %{
5582 base($reg);
5583 index($ireg);
5584 scale(0x0);
5585 disp(0x0);
5586 %}
5587 %}
5588
5589 operand indIndexN(iRegN reg, iRegL lreg)
5590 %{
5591 predicate(CompressedOops::shift() == 0);
5592 constraint(ALLOC_IN_RC(ptr_reg));
5593 match(AddP (DecodeN reg) lreg);
5594 op_cost(0);
5595 format %{ "$reg, $lreg\t# narrow" %}
5596 interface(MEMORY_INTER) %{
5597 base($reg);
5598 index($lreg);
5599 scale(0x0);
5600 disp(0x0);
5601 %}
5602 %}
5603
5604 operand indOffIN(iRegN reg, immIOffset off)
5605 %{
5606 predicate(CompressedOops::shift() == 0);
5607 constraint(ALLOC_IN_RC(ptr_reg));
5608 match(AddP (DecodeN reg) off);
5609 op_cost(0);
5610 format %{ "[$reg, $off]\t# narrow" %}
5611 interface(MEMORY_INTER) %{
5612 base($reg);
5613 index(0xffffffff);
5614 scale(0x0);
5615 disp($off);
5616 %}
5617 %}
5618
5619 operand indOffLN(iRegN reg, immLOffset off)
5620 %{
5621 predicate(CompressedOops::shift() == 0);
5622 constraint(ALLOC_IN_RC(ptr_reg));
5623 match(AddP (DecodeN reg) off);
5624 op_cost(0);
5625 format %{ "[$reg, $off]\t# narrow" %}
5626 interface(MEMORY_INTER) %{
5627 base($reg);
5628 index(0xffffffff);
5629 scale(0x0);
5630 disp($off);
5631 %}
5632 %}
5633
5634
5635 //----------Special Memory Operands--------------------------------------------
5636 // Stack Slot Operand - This operand is used for loading and storing temporary
5637 // values on the stack where a match requires a value to
5638 // flow through memory.
5639 operand stackSlotP(sRegP reg)
5640 %{
5641 constraint(ALLOC_IN_RC(stack_slots));
5642 op_cost(100);
5643 // No match rule because this operand is only generated in matching
5644 // match(RegP);
5645 format %{ "[$reg]" %}
5646 interface(MEMORY_INTER) %{
5647 base(0x1e); // RSP
5648 index(0x0); // No Index
5649 scale(0x0); // No Scale
5650 disp($reg); // Stack Offset
5651 %}
5652 %}
5653
5654 operand stackSlotI(sRegI reg)
5655 %{
5656 constraint(ALLOC_IN_RC(stack_slots));
5657 // No match rule because this operand is only generated in matching
5658 // match(RegI);
5659 format %{ "[$reg]" %}
5660 interface(MEMORY_INTER) %{
5661 base(0x1e); // RSP
5662 index(0x0); // No Index
5663 scale(0x0); // No Scale
5664 disp($reg); // Stack Offset
5665 %}
5666 %}
5667
5668 operand stackSlotF(sRegF reg)
5669 %{
5670 constraint(ALLOC_IN_RC(stack_slots));
5671 // No match rule because this operand is only generated in matching
5672 // match(RegF);
5673 format %{ "[$reg]" %}
5674 interface(MEMORY_INTER) %{
5675 base(0x1e); // RSP
5676 index(0x0); // No Index
5677 scale(0x0); // No Scale
5678 disp($reg); // Stack Offset
5679 %}
5680 %}
5681
5682 operand stackSlotD(sRegD reg)
5683 %{
5684 constraint(ALLOC_IN_RC(stack_slots));
5685 // No match rule because this operand is only generated in matching
5686 // match(RegD);
5687 format %{ "[$reg]" %}
5688 interface(MEMORY_INTER) %{
5689 base(0x1e); // RSP
5690 index(0x0); // No Index
5691 scale(0x0); // No Scale
5692 disp($reg); // Stack Offset
5693 %}
5694 %}
5695
5696 operand stackSlotL(sRegL reg)
5697 %{
5698 constraint(ALLOC_IN_RC(stack_slots));
5699 // No match rule because this operand is only generated in matching
5700 // match(RegL);
5701 format %{ "[$reg]" %}
5702 interface(MEMORY_INTER) %{
5703 base(0x1e); // RSP
5704 index(0x0); // No Index
5705 scale(0x0); // No Scale
5706 disp($reg); // Stack Offset
5707 %}
5708 %}
5709
5710 // Operands for expressing Control Flow
5711 // NOTE: Label is a predefined operand which should not be redefined in
5712 // the AD file. It is generically handled within the ADLC.
5713
5714 //----------Conditional Branch Operands----------------------------------------
5715 // Comparison Op - This is the operation of the comparison, and is limited to
5716 // the following set of codes:
5717 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5718 //
5719 // Other attributes of the comparison, such as unsignedness, are specified
5720 // by the comparison instruction that sets a condition code flags register.
5721 // That result is represented by a flags operand whose subtype is appropriate
5722 // to the unsignedness (etc.) of the comparison.
5723 //
5724 // Later, the instruction which matches both the Comparison Op (a Bool) and
5725 // the flags (produced by the Cmp) specifies the coding of the comparison op
5726 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5727
5728 // used for signed integral comparisons and fp comparisons
5729
5730 operand cmpOp()
5731 %{
5732 match(Bool);
5733
5734 format %{ "" %}
5735 interface(COND_INTER) %{
5736 equal(0x0, "eq");
5737 not_equal(0x1, "ne");
5738 less(0xb, "lt");
5739 greater_equal(0xa, "ge");
5740 less_equal(0xd, "le");
5741 greater(0xc, "gt");
5742 overflow(0x6, "vs");
5743 no_overflow(0x7, "vc");
5744 %}
5745 %}
5746
5747 // used for unsigned integral comparisons
5748
5749 operand cmpOpU()
5750 %{
5751 match(Bool);
5752
5753 format %{ "" %}
5754 interface(COND_INTER) %{
5755 equal(0x0, "eq");
5756 not_equal(0x1, "ne");
5757 less(0x3, "lo");
5758 greater_equal(0x2, "hs");
5759 less_equal(0x9, "ls");
5760 greater(0x8, "hi");
5761 overflow(0x6, "vs");
5762 no_overflow(0x7, "vc");
5763 %}
5764 %}
5765
5766 // used for certain integral comparisons which can be
5767 // converted to cbxx or tbxx instructions
5768
5769 operand cmpOpEqNe()
5770 %{
5771 match(Bool);
5772 op_cost(0);
5773 predicate(n->as_Bool()->_test._test == BoolTest::ne
5774 || n->as_Bool()->_test._test == BoolTest::eq);
5775
5776 format %{ "" %}
5777 interface(COND_INTER) %{
5778 equal(0x0, "eq");
5779 not_equal(0x1, "ne");
5780 less(0xb, "lt");
5781 greater_equal(0xa, "ge");
5782 less_equal(0xd, "le");
5783 greater(0xc, "gt");
5784 overflow(0x6, "vs");
5785 no_overflow(0x7, "vc");
5786 %}
5787 %}
5788
5789 // used for certain integral comparisons which can be
5790 // converted to cbxx or tbxx instructions
5791
5792 operand cmpOpLtGe()
5793 %{
5794 match(Bool);
5795 op_cost(0);
5796
5797 predicate(n->as_Bool()->_test._test == BoolTest::lt
5798 || n->as_Bool()->_test._test == BoolTest::ge);
5799
5800 format %{ "" %}
5801 interface(COND_INTER) %{
5802 equal(0x0, "eq");
5803 not_equal(0x1, "ne");
5804 less(0xb, "lt");
5805 greater_equal(0xa, "ge");
5806 less_equal(0xd, "le");
5807 greater(0xc, "gt");
5808 overflow(0x6, "vs");
5809 no_overflow(0x7, "vc");
5810 %}
5811 %}
5812
5813 // used for certain unsigned integral comparisons which can be
5814 // converted to cbxx or tbxx instructions
5815
5816 operand cmpOpUEqNeLeGt()
5817 %{
5818 match(Bool);
5819 op_cost(0);
5820
5821 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5822 n->as_Bool()->_test._test == BoolTest::ne ||
5823 n->as_Bool()->_test._test == BoolTest::le ||
5824 n->as_Bool()->_test._test == BoolTest::gt);
5825
5826 format %{ "" %}
5827 interface(COND_INTER) %{
5828 equal(0x0, "eq");
5829 not_equal(0x1, "ne");
5830 less(0x3, "lo");
5831 greater_equal(0x2, "hs");
5832 less_equal(0x9, "ls");
5833 greater(0x8, "hi");
5834 overflow(0x6, "vs");
5835 no_overflow(0x7, "vc");
5836 %}
5837 %}
5838
5839 // Special operand allowing long args to int ops to be truncated for free
5840
5841 operand iRegL2I(iRegL reg) %{
5842
5843 op_cost(0);
5844
5845 match(ConvL2I reg);
5846
5847 format %{ "l2i($reg)" %}
5848
5849 interface(REG_INTER)
5850 %}
5851
5852 operand iRegL2P(iRegL reg) %{
5853
5854 op_cost(0);
5855
5856 match(CastX2P reg);
5857
5858 format %{ "l2p($reg)" %}
5859
5860 interface(REG_INTER)
5861 %}
5862
5863 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5864 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5865 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5866 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5867
5868 //----------OPERAND CLASSES----------------------------------------------------
5869 // Operand Classes are groups of operands that are used as to simplify
5870 // instruction definitions by not requiring the AD writer to specify
5871 // separate instructions for every form of operand when the
5872 // instruction accepts multiple operand types with the same basic
5873 // encoding and format. The classic case of this is memory operands.
5874
5875 // memory is used to define read/write location for load/store
5876 // instruction defs. we can turn a memory op into an Address
5877
5878 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5879 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5880
5881 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5882 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5883
5884 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5885 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5886
5887 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5888 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5889
5890 // All of the memory operands. For the pipeline description.
5891 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5892 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5893 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5894
5895
5896 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5897 // operations. it allows the src to be either an iRegI or a (ConvL2I
5898 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5899 // can be elided because the 32-bit instruction will just employ the
5900 // lower 32 bits anyway.
5901 //
5902 // n.b. this does not elide all L2I conversions. if the truncated
5903 // value is consumed by more than one operation then the ConvL2I
5904 // cannot be bundled into the consuming nodes so an l2i gets planted
5905 // (actually a movw $dst $src) and the downstream instructions consume
5906 // the result of the l2i as an iRegI input. That's a shame since the
5907 // movw is actually redundant but its not too costly.
5908
5909 opclass iRegIorL2I(iRegI, iRegL2I);
5910 opclass iRegPorL2P(iRegP, iRegL2P);
5911
5912 //----------PIPELINE-----------------------------------------------------------
5913 // Rules which define the behavior of the target architectures pipeline.
5914
5915 // For specific pipelines, eg A53, define the stages of that pipeline
5916 //pipe_desc(ISS, EX1, EX2, WR);
5917 #define ISS S0
5918 #define EX1 S1
5919 #define EX2 S2
5920 #define WR S3
5921
5922 // Integer ALU reg operation
5923 pipeline %{
5924
5925 attributes %{
5926 // ARM instructions are of fixed length
5927 fixed_size_instructions; // Fixed size instructions TODO does
5928 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5929 // ARM instructions come in 32-bit word units
5930 instruction_unit_size = 4; // An instruction is 4 bytes long
5931 instruction_fetch_unit_size = 64; // The processor fetches one line
5932 instruction_fetch_units = 1; // of 64 bytes
5933 %}
5934
5935 // We don't use an actual pipeline model so don't care about resources
5936 // or description. we do use pipeline classes to introduce fixed
5937 // latencies
5938
5939 //----------RESOURCES----------------------------------------------------------
5940 // Resources are the functional units available to the machine
5941
5942 resources( INS0, INS1, INS01 = INS0 | INS1,
5943 ALU0, ALU1, ALU = ALU0 | ALU1,
5944 MAC,
5945 DIV,
5946 BRANCH,
5947 LDST,
5948 NEON_FP);
5949
5950 //----------PIPELINE DESCRIPTION-----------------------------------------------
5951 // Pipeline Description specifies the stages in the machine's pipeline
5952
5953 // Define the pipeline as a generic 6 stage pipeline
5954 pipe_desc(S0, S1, S2, S3, S4, S5);
5955
5956 //----------PIPELINE CLASSES---------------------------------------------------
5957 // Pipeline Classes describe the stages in which input and output are
5958 // referenced by the hardware pipeline.
5959
5960 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5961 %{
5962 single_instruction;
5963 src1 : S1(read);
5964 src2 : S2(read);
5965 dst : S5(write);
5966 INS01 : ISS;
5967 NEON_FP : S5;
5968 %}
5969
5970 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5971 %{
5972 single_instruction;
5973 src1 : S1(read);
5974 src2 : S2(read);
5975 dst : S5(write);
5976 INS01 : ISS;
5977 NEON_FP : S5;
5978 %}
5979
5980 pipe_class fp_uop_s(vRegF dst, vRegF src)
5981 %{
5982 single_instruction;
5983 src : S1(read);
5984 dst : S5(write);
5985 INS01 : ISS;
5986 NEON_FP : S5;
5987 %}
5988
5989 pipe_class fp_uop_d(vRegD dst, vRegD src)
5990 %{
5991 single_instruction;
5992 src : S1(read);
5993 dst : S5(write);
5994 INS01 : ISS;
5995 NEON_FP : S5;
5996 %}
5997
5998 pipe_class fp_d2f(vRegF dst, vRegD src)
5999 %{
6000 single_instruction;
6001 src : S1(read);
6002 dst : S5(write);
6003 INS01 : ISS;
6004 NEON_FP : S5;
6005 %}
6006
6007 pipe_class fp_f2d(vRegD dst, vRegF src)
6008 %{
6009 single_instruction;
6010 src : S1(read);
6011 dst : S5(write);
6012 INS01 : ISS;
6013 NEON_FP : S5;
6014 %}
6015
6016 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6017 %{
6018 single_instruction;
6019 src : S1(read);
6020 dst : S5(write);
6021 INS01 : ISS;
6022 NEON_FP : S5;
6023 %}
6024
6025 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6026 %{
6027 single_instruction;
6028 src : S1(read);
6029 dst : S5(write);
6030 INS01 : ISS;
6031 NEON_FP : S5;
6032 %}
6033
6034 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6035 %{
6036 single_instruction;
6037 src : S1(read);
6038 dst : S5(write);
6039 INS01 : ISS;
6040 NEON_FP : S5;
6041 %}
6042
6043 pipe_class fp_l2f(vRegF dst, iRegL src)
6044 %{
6045 single_instruction;
6046 src : S1(read);
6047 dst : S5(write);
6048 INS01 : ISS;
6049 NEON_FP : S5;
6050 %}
6051
6052 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6053 %{
6054 single_instruction;
6055 src : S1(read);
6056 dst : S5(write);
6057 INS01 : ISS;
6058 NEON_FP : S5;
6059 %}
6060
6061 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6062 %{
6063 single_instruction;
6064 src : S1(read);
6065 dst : S5(write);
6066 INS01 : ISS;
6067 NEON_FP : S5;
6068 %}
6069
6070 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6071 %{
6072 single_instruction;
6073 src : S1(read);
6074 dst : S5(write);
6075 INS01 : ISS;
6076 NEON_FP : S5;
6077 %}
6078
6079 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6080 %{
6081 single_instruction;
6082 src : S1(read);
6083 dst : S5(write);
6084 INS01 : ISS;
6085 NEON_FP : S5;
6086 %}
6087
6088 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6089 %{
6090 single_instruction;
6091 src1 : S1(read);
6092 src2 : S2(read);
6093 dst : S5(write);
6094 INS0 : ISS;
6095 NEON_FP : S5;
6096 %}
6097
6098 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6099 %{
6100 single_instruction;
6101 src1 : S1(read);
6102 src2 : S2(read);
6103 dst : S5(write);
6104 INS0 : ISS;
6105 NEON_FP : S5;
6106 %}
6107
6108 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6109 %{
6110 single_instruction;
6111 cr : S1(read);
6112 src1 : S1(read);
6113 src2 : S1(read);
6114 dst : S3(write);
6115 INS01 : ISS;
6116 NEON_FP : S3;
6117 %}
6118
6119 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6120 %{
6121 single_instruction;
6122 cr : S1(read);
6123 src1 : S1(read);
6124 src2 : S1(read);
6125 dst : S3(write);
6126 INS01 : ISS;
6127 NEON_FP : S3;
6128 %}
6129
6130 pipe_class fp_imm_s(vRegF dst)
6131 %{
6132 single_instruction;
6133 dst : S3(write);
6134 INS01 : ISS;
6135 NEON_FP : S3;
6136 %}
6137
6138 pipe_class fp_imm_d(vRegD dst)
6139 %{
6140 single_instruction;
6141 dst : S3(write);
6142 INS01 : ISS;
6143 NEON_FP : S3;
6144 %}
6145
6146 pipe_class fp_load_constant_s(vRegF dst)
6147 %{
6148 single_instruction;
6149 dst : S4(write);
6150 INS01 : ISS;
6151 NEON_FP : S4;
6152 %}
6153
6154 pipe_class fp_load_constant_d(vRegD dst)
6155 %{
6156 single_instruction;
6157 dst : S4(write);
6158 INS01 : ISS;
6159 NEON_FP : S4;
6160 %}
6161
6162 //------- Integer ALU operations --------------------------
6163
6164 // Integer ALU reg-reg operation
6165 // Operands needed in EX1, result generated in EX2
6166 // Eg. ADD x0, x1, x2
6167 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6168 %{
6169 single_instruction;
6170 dst : EX2(write);
6171 src1 : EX1(read);
6172 src2 : EX1(read);
6173 INS01 : ISS; // Dual issue as instruction 0 or 1
6174 ALU : EX2;
6175 %}
6176
6177 // Integer ALU reg-reg operation with constant shift
6178 // Shifted register must be available in LATE_ISS instead of EX1
6179 // Eg. ADD x0, x1, x2, LSL #2
6180 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6181 %{
6182 single_instruction;
6183 dst : EX2(write);
6184 src1 : EX1(read);
6185 src2 : ISS(read);
6186 INS01 : ISS;
6187 ALU : EX2;
6188 %}
6189
6190 // Integer ALU reg operation with constant shift
6191 // Eg. LSL x0, x1, #shift
6192 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6193 %{
6194 single_instruction;
6195 dst : EX2(write);
6196 src1 : ISS(read);
6197 INS01 : ISS;
6198 ALU : EX2;
6199 %}
6200
6201 // Integer ALU reg-reg operation with variable shift
6202 // Both operands must be available in LATE_ISS instead of EX1
6203 // Result is available in EX1 instead of EX2
6204 // Eg. LSLV x0, x1, x2
6205 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6206 %{
6207 single_instruction;
6208 dst : EX1(write);
6209 src1 : ISS(read);
6210 src2 : ISS(read);
6211 INS01 : ISS;
6212 ALU : EX1;
6213 %}
6214
6215 // Integer ALU reg-reg operation with extract
6216 // As for _vshift above, but result generated in EX2
6217 // Eg. EXTR x0, x1, x2, #N
6218 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6219 %{
6220 single_instruction;
6221 dst : EX2(write);
6222 src1 : ISS(read);
6223 src2 : ISS(read);
6224 INS1 : ISS; // Can only dual issue as Instruction 1
6225 ALU : EX1;
6226 %}
6227
6228 // Integer ALU reg operation
6229 // Eg. NEG x0, x1
6230 pipe_class ialu_reg(iRegI dst, iRegI src)
6231 %{
6232 single_instruction;
6233 dst : EX2(write);
6234 src : EX1(read);
6235 INS01 : ISS;
6236 ALU : EX2;
6237 %}
6238
6239 // Integer ALU reg mmediate operation
6240 // Eg. ADD x0, x1, #N
6241 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6242 %{
6243 single_instruction;
6244 dst : EX2(write);
6245 src1 : EX1(read);
6246 INS01 : ISS;
6247 ALU : EX2;
6248 %}
6249
6250 // Integer ALU immediate operation (no source operands)
6251 // Eg. MOV x0, #N
6252 pipe_class ialu_imm(iRegI dst)
6253 %{
6254 single_instruction;
6255 dst : EX1(write);
6256 INS01 : ISS;
6257 ALU : EX1;
6258 %}
6259
6260 //------- Compare operation -------------------------------
6261
6262 // Compare reg-reg
6263 // Eg. CMP x0, x1
6264 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6265 %{
6266 single_instruction;
6267 // fixed_latency(16);
6268 cr : EX2(write);
6269 op1 : EX1(read);
6270 op2 : EX1(read);
6271 INS01 : ISS;
6272 ALU : EX2;
6273 %}
6274
6275 // Compare reg-reg
6276 // Eg. CMP x0, #N
6277 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6278 %{
6279 single_instruction;
6280 // fixed_latency(16);
6281 cr : EX2(write);
6282 op1 : EX1(read);
6283 INS01 : ISS;
6284 ALU : EX2;
6285 %}
6286
6287 //------- Conditional instructions ------------------------
6288
6289 // Conditional no operands
6290 // Eg. CSINC x0, zr, zr, <cond>
6291 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6292 %{
6293 single_instruction;
6294 cr : EX1(read);
6295 dst : EX2(write);
6296 INS01 : ISS;
6297 ALU : EX2;
6298 %}
6299
6300 // Conditional 2 operand
6301 // EG. CSEL X0, X1, X2, <cond>
6302 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6303 %{
6304 single_instruction;
6305 cr : EX1(read);
6306 src1 : EX1(read);
6307 src2 : EX1(read);
6308 dst : EX2(write);
6309 INS01 : ISS;
6310 ALU : EX2;
6311 %}
6312
6313 // Conditional 2 operand
6314 // EG. CSEL X0, X1, X2, <cond>
6315 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6316 %{
6317 single_instruction;
6318 cr : EX1(read);
6319 src : EX1(read);
6320 dst : EX2(write);
6321 INS01 : ISS;
6322 ALU : EX2;
6323 %}
6324
6325 //------- Multiply pipeline operations --------------------
6326
6327 // Multiply reg-reg
6328 // Eg. MUL w0, w1, w2
6329 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6330 %{
6331 single_instruction;
6332 dst : WR(write);
6333 src1 : ISS(read);
6334 src2 : ISS(read);
6335 INS01 : ISS;
6336 MAC : WR;
6337 %}
6338
6339 // Multiply accumulate
6340 // Eg. MADD w0, w1, w2, w3
6341 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6342 %{
6343 single_instruction;
6344 dst : WR(write);
6345 src1 : ISS(read);
6346 src2 : ISS(read);
6347 src3 : ISS(read);
6348 INS01 : ISS;
6349 MAC : WR;
6350 %}
6351
6352 // Eg. MUL w0, w1, w2
6353 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6354 %{
6355 single_instruction;
6356 fixed_latency(3); // Maximum latency for 64 bit mul
6357 dst : WR(write);
6358 src1 : ISS(read);
6359 src2 : ISS(read);
6360 INS01 : ISS;
6361 MAC : WR;
6362 %}
6363
6364 // Multiply accumulate
6365 // Eg. MADD w0, w1, w2, w3
6366 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6367 %{
6368 single_instruction;
6369 fixed_latency(3); // Maximum latency for 64 bit mul
6370 dst : WR(write);
6371 src1 : ISS(read);
6372 src2 : ISS(read);
6373 src3 : ISS(read);
6374 INS01 : ISS;
6375 MAC : WR;
6376 %}
6377
6378 //------- Divide pipeline operations --------------------
6379
6380 // Eg. SDIV w0, w1, w2
6381 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6382 %{
6383 single_instruction;
6384 fixed_latency(8); // Maximum latency for 32 bit divide
6385 dst : WR(write);
6386 src1 : ISS(read);
6387 src2 : ISS(read);
6388 INS0 : ISS; // Can only dual issue as instruction 0
6389 DIV : WR;
6390 %}
6391
6392 // Eg. SDIV x0, x1, x2
6393 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6394 %{
6395 single_instruction;
6396 fixed_latency(16); // Maximum latency for 64 bit divide
6397 dst : WR(write);
6398 src1 : ISS(read);
6399 src2 : ISS(read);
6400 INS0 : ISS; // Can only dual issue as instruction 0
6401 DIV : WR;
6402 %}
6403
6404 //------- Load pipeline operations ------------------------
6405
6406 // Load - prefetch
6407 // Eg. PFRM <mem>
6408 pipe_class iload_prefetch(memory mem)
6409 %{
6410 single_instruction;
6411 mem : ISS(read);
6412 INS01 : ISS;
6413 LDST : WR;
6414 %}
6415
6416 // Load - reg, mem
6417 // Eg. LDR x0, <mem>
6418 pipe_class iload_reg_mem(iRegI dst, memory mem)
6419 %{
6420 single_instruction;
6421 dst : WR(write);
6422 mem : ISS(read);
6423 INS01 : ISS;
6424 LDST : WR;
6425 %}
6426
6427 // Load - reg, reg
6428 // Eg. LDR x0, [sp, x1]
6429 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6430 %{
6431 single_instruction;
6432 dst : WR(write);
6433 src : ISS(read);
6434 INS01 : ISS;
6435 LDST : WR;
6436 %}
6437
6438 //------- Store pipeline operations -----------------------
6439
6440 // Store - zr, mem
6441 // Eg. STR zr, <mem>
6442 pipe_class istore_mem(memory mem)
6443 %{
6444 single_instruction;
6445 mem : ISS(read);
6446 INS01 : ISS;
6447 LDST : WR;
6448 %}
6449
6450 // Store - reg, mem
6451 // Eg. STR x0, <mem>
6452 pipe_class istore_reg_mem(iRegI src, memory mem)
6453 %{
6454 single_instruction;
6455 mem : ISS(read);
6456 src : EX2(read);
6457 INS01 : ISS;
6458 LDST : WR;
6459 %}
6460
6461 // Store - reg, reg
6462 // Eg. STR x0, [sp, x1]
6463 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6464 %{
6465 single_instruction;
6466 dst : ISS(read);
6467 src : EX2(read);
6468 INS01 : ISS;
6469 LDST : WR;
6470 %}
6471
6472 //------- Store pipeline operations -----------------------
6473
6474 // Branch
6475 pipe_class pipe_branch()
6476 %{
6477 single_instruction;
6478 INS01 : ISS;
6479 BRANCH : EX1;
6480 %}
6481
6482 // Conditional branch
6483 pipe_class pipe_branch_cond(rFlagsReg cr)
6484 %{
6485 single_instruction;
6486 cr : EX1(read);
6487 INS01 : ISS;
6488 BRANCH : EX1;
6489 %}
6490
6491 // Compare & Branch
6492 // EG. CBZ/CBNZ
6493 pipe_class pipe_cmp_branch(iRegI op1)
6494 %{
6495 single_instruction;
6496 op1 : EX1(read);
6497 INS01 : ISS;
6498 BRANCH : EX1;
6499 %}
6500
6501 //------- Synchronisation operations ----------------------
6502
6503 // Any operation requiring serialization.
6504 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6505 pipe_class pipe_serial()
6506 %{
6507 single_instruction;
6508 force_serialization;
6509 fixed_latency(16);
6510 INS01 : ISS(2); // Cannot dual issue with any other instruction
6511 LDST : WR;
6512 %}
6513
6514 // Generic big/slow expanded idiom - also serialized
6515 pipe_class pipe_slow()
6516 %{
6517 instruction_count(10);
6518 multiple_bundles;
6519 force_serialization;
6520 fixed_latency(16);
6521 INS01 : ISS(2); // Cannot dual issue with any other instruction
6522 LDST : WR;
6523 %}
6524
6525 // Empty pipeline class
6526 pipe_class pipe_class_empty()
6527 %{
6528 single_instruction;
6529 fixed_latency(0);
6530 %}
6531
6532 // Default pipeline class.
6533 pipe_class pipe_class_default()
6534 %{
6535 single_instruction;
6536 fixed_latency(2);
6537 %}
6538
6539 // Pipeline class for compares.
6540 pipe_class pipe_class_compare()
6541 %{
6542 single_instruction;
6543 fixed_latency(16);
6544 %}
6545
6546 // Pipeline class for memory operations.
6547 pipe_class pipe_class_memory()
6548 %{
6549 single_instruction;
6550 fixed_latency(16);
6551 %}
6552
6553 // Pipeline class for call.
6554 pipe_class pipe_class_call()
6555 %{
6556 single_instruction;
6557 fixed_latency(100);
6558 %}
6559
6560 // Define the class for the Nop node.
6561 define %{
6562 MachNop = pipe_class_empty;
6563 %}
6564
6565 %}
6566 //----------INSTRUCTIONS-------------------------------------------------------
6567 //
6568 // match -- States which machine-independent subtree may be replaced
6569 // by this instruction.
6570 // ins_cost -- The estimated cost of this instruction is used by instruction
6571 // selection to identify a minimum cost tree of machine
6572 // instructions that matches a tree of machine-independent
6573 // instructions.
6574 // format -- A string providing the disassembly for this instruction.
6575 // The value of an instruction's operand may be inserted
6576 // by referring to it with a '$' prefix.
6577 // opcode -- Three instruction opcodes may be provided. These are referred
6578 // to within an encode class as $primary, $secondary, and $tertiary
6579 // rrspectively. The primary opcode is commonly used to
6580 // indicate the type of machine instruction, while secondary
6581 // and tertiary are often used for prefix options or addressing
6582 // modes.
6583 // ins_encode -- A list of encode classes with parameters. The encode class
6584 // name must have been defined in an 'enc_class' specification
6585 // in the encode section of the architecture description.
6586
6587 // ============================================================================
6588 // Memory (Load/Store) Instructions
6589
6590 // Load Instructions
6591
6592 // Load Byte (8 bit signed)
6593 instruct loadB(iRegINoSp dst, memory1 mem)
6594 %{
6595 match(Set dst (LoadB mem));
6596 predicate(!needs_acquiring_load(n));
6597
6598 ins_cost(4 * INSN_COST);
6599 format %{ "ldrsbw $dst, $mem\t# byte" %}
6600
6601 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6602
6603 ins_pipe(iload_reg_mem);
6604 %}
6605
6606 // Load Byte (8 bit signed) into long
6607 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6608 %{
6609 match(Set dst (ConvI2L (LoadB mem)));
6610 predicate(!needs_acquiring_load(n->in(1)));
6611
6612 ins_cost(4 * INSN_COST);
6613 format %{ "ldrsb $dst, $mem\t# byte" %}
6614
6615 ins_encode(aarch64_enc_ldrsb(dst, mem));
6616
6617 ins_pipe(iload_reg_mem);
6618 %}
6619
6620 // Load Byte (8 bit unsigned)
6621 instruct loadUB(iRegINoSp dst, memory1 mem)
6622 %{
6623 match(Set dst (LoadUB mem));
6624 predicate(!needs_acquiring_load(n));
6625
6626 ins_cost(4 * INSN_COST);
6627 format %{ "ldrbw $dst, $mem\t# byte" %}
6628
6629 ins_encode(aarch64_enc_ldrb(dst, mem));
6630
6631 ins_pipe(iload_reg_mem);
6632 %}
6633
6634 // Load Byte (8 bit unsigned) into long
6635 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6636 %{
6637 match(Set dst (ConvI2L (LoadUB mem)));
6638 predicate(!needs_acquiring_load(n->in(1)));
6639
6640 ins_cost(4 * INSN_COST);
6641 format %{ "ldrb $dst, $mem\t# byte" %}
6642
6643 ins_encode(aarch64_enc_ldrb(dst, mem));
6644
6645 ins_pipe(iload_reg_mem);
6646 %}
6647
6648 // Load Short (16 bit signed)
6649 instruct loadS(iRegINoSp dst, memory2 mem)
6650 %{
6651 match(Set dst (LoadS mem));
6652 predicate(!needs_acquiring_load(n));
6653
6654 ins_cost(4 * INSN_COST);
6655 format %{ "ldrshw $dst, $mem\t# short" %}
6656
6657 ins_encode(aarch64_enc_ldrshw(dst, mem));
6658
6659 ins_pipe(iload_reg_mem);
6660 %}
6661
6662 // Load Short (16 bit signed) into long
6663 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6664 %{
6665 match(Set dst (ConvI2L (LoadS mem)));
6666 predicate(!needs_acquiring_load(n->in(1)));
6667
6668 ins_cost(4 * INSN_COST);
6669 format %{ "ldrsh $dst, $mem\t# short" %}
6670
6671 ins_encode(aarch64_enc_ldrsh(dst, mem));
6672
6673 ins_pipe(iload_reg_mem);
6674 %}
6675
6676 // Load Char (16 bit unsigned)
6677 instruct loadUS(iRegINoSp dst, memory2 mem)
6678 %{
6679 match(Set dst (LoadUS mem));
6680 predicate(!needs_acquiring_load(n));
6681
6682 ins_cost(4 * INSN_COST);
6683 format %{ "ldrh $dst, $mem\t# short" %}
6684
6685 ins_encode(aarch64_enc_ldrh(dst, mem));
6686
6687 ins_pipe(iload_reg_mem);
6688 %}
6689
6690 // Load Short/Char (16 bit unsigned) into long
6691 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6692 %{
6693 match(Set dst (ConvI2L (LoadUS mem)));
6694 predicate(!needs_acquiring_load(n->in(1)));
6695
6696 ins_cost(4 * INSN_COST);
6697 format %{ "ldrh $dst, $mem\t# short" %}
6698
6699 ins_encode(aarch64_enc_ldrh(dst, mem));
6700
6701 ins_pipe(iload_reg_mem);
6702 %}
6703
6704 // Load Integer (32 bit signed)
6705 instruct loadI(iRegINoSp dst, memory4 mem)
6706 %{
6707 match(Set dst (LoadI mem));
6708 predicate(!needs_acquiring_load(n));
6709
6710 ins_cost(4 * INSN_COST);
6711 format %{ "ldrw $dst, $mem\t# int" %}
6712
6713 ins_encode(aarch64_enc_ldrw(dst, mem));
6714
6715 ins_pipe(iload_reg_mem);
6716 %}
6717
6718 // Load Integer (32 bit signed) into long
6719 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6720 %{
6721 match(Set dst (ConvI2L (LoadI mem)));
6722 predicate(!needs_acquiring_load(n->in(1)));
6723
6724 ins_cost(4 * INSN_COST);
6725 format %{ "ldrsw $dst, $mem\t# int" %}
6726
6727 ins_encode(aarch64_enc_ldrsw(dst, mem));
6728
6729 ins_pipe(iload_reg_mem);
6730 %}
6731
6732 // Load Integer (32 bit unsigned) into long
6733 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6734 %{
6735 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6736 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6737
6738 ins_cost(4 * INSN_COST);
6739 format %{ "ldrw $dst, $mem\t# int" %}
6740
6741 ins_encode(aarch64_enc_ldrw(dst, mem));
6742
6743 ins_pipe(iload_reg_mem);
6744 %}
6745
6746 // Load Long (64 bit signed)
6747 instruct loadL(iRegLNoSp dst, memory8 mem)
6748 %{
6749 match(Set dst (LoadL mem));
6750 predicate(!needs_acquiring_load(n));
6751
6752 ins_cost(4 * INSN_COST);
6753 format %{ "ldr $dst, $mem\t# int" %}
6754
6755 ins_encode(aarch64_enc_ldr(dst, mem));
6756
6757 ins_pipe(iload_reg_mem);
6758 %}
6759
6760 // Load Range
6761 instruct loadRange(iRegINoSp dst, memory4 mem)
6762 %{
6763 match(Set dst (LoadRange mem));
6764
6765 ins_cost(4 * INSN_COST);
6766 format %{ "ldrw $dst, $mem\t# range" %}
6767
6768 ins_encode(aarch64_enc_ldrw(dst, mem));
6769
6770 ins_pipe(iload_reg_mem);
6771 %}
6772
6773 // Load Pointer
6774 instruct loadP(iRegPNoSp dst, memory8 mem)
6775 %{
6776 match(Set dst (LoadP mem));
6777 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6778
6779 ins_cost(4 * INSN_COST);
6780 format %{ "ldr $dst, $mem\t# ptr" %}
6781
6782 ins_encode(aarch64_enc_ldr(dst, mem));
6783
6784 ins_pipe(iload_reg_mem);
6785 %}
6786
6787 // Load Compressed Pointer
6788 instruct loadN(iRegNNoSp dst, memory4 mem)
6789 %{
6790 match(Set dst (LoadN mem));
6791 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6792
6793 ins_cost(4 * INSN_COST);
6794 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6795
6796 ins_encode(aarch64_enc_ldrw(dst, mem));
6797
6798 ins_pipe(iload_reg_mem);
6799 %}
6800
6801 // Load Klass Pointer
6802 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6803 %{
6804 match(Set dst (LoadKlass mem));
6805 predicate(!needs_acquiring_load(n));
6806
6807 ins_cost(4 * INSN_COST);
6808 format %{ "ldr $dst, $mem\t# class" %}
6809
6810 ins_encode(aarch64_enc_ldr(dst, mem));
6811
6812 ins_pipe(iload_reg_mem);
6813 %}
6814
6815 // Load Narrow Klass Pointer
6816 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6817 %{
6818 match(Set dst (LoadNKlass mem));
6819 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6820
6821 ins_cost(4 * INSN_COST);
6822 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6823
6824 ins_encode(aarch64_enc_ldrw(dst, mem));
6825
6826 ins_pipe(iload_reg_mem);
6827 %}
6828
6829 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6830 %{
6831 match(Set dst (LoadNKlass mem));
6832 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6833
6834 ins_cost(4 * INSN_COST);
6835 format %{
6836 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6837 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6838 %}
6839 ins_encode %{
6840 // inlined aarch64_enc_ldrw
6841 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6842 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6843 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6844 %}
6845 ins_pipe(iload_reg_mem);
6846 %}
6847
6848 // Load Float
6849 instruct loadF(vRegF dst, memory4 mem)
6850 %{
6851 match(Set dst (LoadF mem));
6852 predicate(!needs_acquiring_load(n));
6853
6854 ins_cost(4 * INSN_COST);
6855 format %{ "ldrs $dst, $mem\t# float" %}
6856
6857 ins_encode( aarch64_enc_ldrs(dst, mem) );
6858
6859 ins_pipe(pipe_class_memory);
6860 %}
6861
6862 // Load Double
6863 instruct loadD(vRegD dst, memory8 mem)
6864 %{
6865 match(Set dst (LoadD mem));
6866 predicate(!needs_acquiring_load(n));
6867
6868 ins_cost(4 * INSN_COST);
6869 format %{ "ldrd $dst, $mem\t# double" %}
6870
6871 ins_encode( aarch64_enc_ldrd(dst, mem) );
6872
6873 ins_pipe(pipe_class_memory);
6874 %}
6875
6876
6877 // Load Int Constant
6878 instruct loadConI(iRegINoSp dst, immI src)
6879 %{
6880 match(Set dst src);
6881
6882 ins_cost(INSN_COST);
6883 format %{ "mov $dst, $src\t# int" %}
6884
6885 ins_encode( aarch64_enc_movw_imm(dst, src) );
6886
6887 ins_pipe(ialu_imm);
6888 %}
6889
6890 // Load Long Constant
6891 instruct loadConL(iRegLNoSp dst, immL src)
6892 %{
6893 match(Set dst src);
6894
6895 ins_cost(INSN_COST);
6896 format %{ "mov $dst, $src\t# long" %}
6897
6898 ins_encode( aarch64_enc_mov_imm(dst, src) );
6899
6900 ins_pipe(ialu_imm);
6901 %}
6902
6903 // Load Pointer Constant
6904
6905 instruct loadConP(iRegPNoSp dst, immP con)
6906 %{
6907 match(Set dst con);
6908
6909 ins_cost(INSN_COST * 4);
6910 format %{
6911 "mov $dst, $con\t# ptr\n\t"
6912 %}
6913
6914 ins_encode(aarch64_enc_mov_p(dst, con));
6915
6916 ins_pipe(ialu_imm);
6917 %}
6918
6919 // Load Null Pointer Constant
6920
6921 instruct loadConP0(iRegPNoSp dst, immP0 con)
6922 %{
6923 match(Set dst con);
6924
6925 ins_cost(INSN_COST);
6926 format %{ "mov $dst, $con\t# nullptr ptr" %}
6927
6928 ins_encode(aarch64_enc_mov_p0(dst, con));
6929
6930 ins_pipe(ialu_imm);
6931 %}
6932
6933 // Load Pointer Constant One
6934
6935 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6936 %{
6937 match(Set dst con);
6938
6939 ins_cost(INSN_COST);
6940 format %{ "mov $dst, $con\t# nullptr ptr" %}
6941
6942 ins_encode(aarch64_enc_mov_p1(dst, con));
6943
6944 ins_pipe(ialu_imm);
6945 %}
6946
6947 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6948 %{
6949 match(Set dst con);
6950
6951 ins_cost(INSN_COST);
6952 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6953
6954 ins_encode %{
6955 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6956 %}
6957
6958 ins_pipe(ialu_imm);
6959 %}
6960
6961 // Load Narrow Pointer Constant
6962
6963 instruct loadConN(iRegNNoSp dst, immN con)
6964 %{
6965 match(Set dst con);
6966
6967 ins_cost(INSN_COST * 4);
6968 format %{ "mov $dst, $con\t# compressed ptr" %}
6969
6970 ins_encode(aarch64_enc_mov_n(dst, con));
6971
6972 ins_pipe(ialu_imm);
6973 %}
6974
6975 // Load Narrow Null Pointer Constant
6976
6977 instruct loadConN0(iRegNNoSp dst, immN0 con)
6978 %{
6979 match(Set dst con);
6980
6981 ins_cost(INSN_COST);
6982 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6983
6984 ins_encode(aarch64_enc_mov_n0(dst, con));
6985
6986 ins_pipe(ialu_imm);
6987 %}
6988
6989 // Load Narrow Klass Constant
6990
6991 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6992 %{
6993 match(Set dst con);
6994
6995 ins_cost(INSN_COST);
6996 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6997
6998 ins_encode(aarch64_enc_mov_nk(dst, con));
6999
7000 ins_pipe(ialu_imm);
7001 %}
7002
7003 // Load Packed Float Constant
7004
7005 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7006 match(Set dst con);
7007 ins_cost(INSN_COST * 4);
7008 format %{ "fmovs $dst, $con"%}
7009 ins_encode %{
7010 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7011 %}
7012
7013 ins_pipe(fp_imm_s);
7014 %}
7015
7016 // Load Float Constant
7017
7018 instruct loadConF(vRegF dst, immF con) %{
7019 match(Set dst con);
7020
7021 ins_cost(INSN_COST * 4);
7022
7023 format %{
7024 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7025 %}
7026
7027 ins_encode %{
7028 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7029 %}
7030
7031 ins_pipe(fp_load_constant_s);
7032 %}
7033
7034 // Load Packed Double Constant
7035
7036 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7037 match(Set dst con);
7038 ins_cost(INSN_COST);
7039 format %{ "fmovd $dst, $con"%}
7040 ins_encode %{
7041 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7042 %}
7043
7044 ins_pipe(fp_imm_d);
7045 %}
7046
7047 // Load Double Constant
7048
7049 instruct loadConD(vRegD dst, immD con) %{
7050 match(Set dst con);
7051
7052 ins_cost(INSN_COST * 5);
7053 format %{
7054 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7055 %}
7056
7057 ins_encode %{
7058 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7059 %}
7060
7061 ins_pipe(fp_load_constant_d);
7062 %}
7063
7064 // Load Half Float Constant
7065 instruct loadConH(vRegF dst, immH con) %{
7066 match(Set dst con);
7067 format %{ "mov rscratch1, $con\n\t"
7068 "fmov $dst, rscratch1"
7069 %}
7070 ins_encode %{
7071 __ movw(rscratch1, (uint32_t)$con$$constant);
7072 __ fmovs($dst$$FloatRegister, rscratch1);
7073 %}
7074 ins_pipe(pipe_class_default);
7075 %}
7076
7077 // Store Instructions
7078
7079 // Store Byte
7080 instruct storeB(iRegIorL2I src, memory1 mem)
7081 %{
7082 match(Set mem (StoreB mem src));
7083 predicate(!needs_releasing_store(n));
7084
7085 ins_cost(INSN_COST);
7086 format %{ "strb $src, $mem\t# byte" %}
7087
7088 ins_encode(aarch64_enc_strb(src, mem));
7089
7090 ins_pipe(istore_reg_mem);
7091 %}
7092
7093
7094 instruct storeimmB0(immI0 zero, memory1 mem)
7095 %{
7096 match(Set mem (StoreB mem zero));
7097 predicate(!needs_releasing_store(n));
7098
7099 ins_cost(INSN_COST);
7100 format %{ "strb rscractch2, $mem\t# byte" %}
7101
7102 ins_encode(aarch64_enc_strb0(mem));
7103
7104 ins_pipe(istore_mem);
7105 %}
7106
7107 // Store Char/Short
7108 instruct storeC(iRegIorL2I src, memory2 mem)
7109 %{
7110 match(Set mem (StoreC mem src));
7111 predicate(!needs_releasing_store(n));
7112
7113 ins_cost(INSN_COST);
7114 format %{ "strh $src, $mem\t# short" %}
7115
7116 ins_encode(aarch64_enc_strh(src, mem));
7117
7118 ins_pipe(istore_reg_mem);
7119 %}
7120
7121 instruct storeimmC0(immI0 zero, memory2 mem)
7122 %{
7123 match(Set mem (StoreC mem zero));
7124 predicate(!needs_releasing_store(n));
7125
7126 ins_cost(INSN_COST);
7127 format %{ "strh zr, $mem\t# short" %}
7128
7129 ins_encode(aarch64_enc_strh0(mem));
7130
7131 ins_pipe(istore_mem);
7132 %}
7133
7134 // Store Integer
7135
7136 instruct storeI(iRegIorL2I src, memory4 mem)
7137 %{
7138 match(Set mem(StoreI mem src));
7139 predicate(!needs_releasing_store(n));
7140
7141 ins_cost(INSN_COST);
7142 format %{ "strw $src, $mem\t# int" %}
7143
7144 ins_encode(aarch64_enc_strw(src, mem));
7145
7146 ins_pipe(istore_reg_mem);
7147 %}
7148
7149 instruct storeimmI0(immI0 zero, memory4 mem)
7150 %{
7151 match(Set mem(StoreI mem zero));
7152 predicate(!needs_releasing_store(n));
7153
7154 ins_cost(INSN_COST);
7155 format %{ "strw zr, $mem\t# int" %}
7156
7157 ins_encode(aarch64_enc_strw0(mem));
7158
7159 ins_pipe(istore_mem);
7160 %}
7161
7162 // Store Long (64 bit signed)
7163 instruct storeL(iRegL src, memory8 mem)
7164 %{
7165 match(Set mem (StoreL mem src));
7166 predicate(!needs_releasing_store(n));
7167
7168 ins_cost(INSN_COST);
7169 format %{ "str $src, $mem\t# int" %}
7170
7171 ins_encode(aarch64_enc_str(src, mem));
7172
7173 ins_pipe(istore_reg_mem);
7174 %}
7175
7176 // Store Long (64 bit signed)
7177 instruct storeimmL0(immL0 zero, memory8 mem)
7178 %{
7179 match(Set mem (StoreL mem zero));
7180 predicate(!needs_releasing_store(n));
7181
7182 ins_cost(INSN_COST);
7183 format %{ "str zr, $mem\t# int" %}
7184
7185 ins_encode(aarch64_enc_str0(mem));
7186
7187 ins_pipe(istore_mem);
7188 %}
7189
7190 // Store Pointer
7191 instruct storeP(iRegP src, memory8 mem)
7192 %{
7193 match(Set mem (StoreP mem src));
7194 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7195
7196 ins_cost(INSN_COST);
7197 format %{ "str $src, $mem\t# ptr" %}
7198
7199 ins_encode(aarch64_enc_str(src, mem));
7200
7201 ins_pipe(istore_reg_mem);
7202 %}
7203
7204 // Store Pointer
7205 instruct storeimmP0(immP0 zero, memory8 mem)
7206 %{
7207 match(Set mem (StoreP mem zero));
7208 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7209
7210 ins_cost(INSN_COST);
7211 format %{ "str zr, $mem\t# ptr" %}
7212
7213 ins_encode(aarch64_enc_str0(mem));
7214
7215 ins_pipe(istore_mem);
7216 %}
7217
7218 // Store Compressed Pointer
7219 instruct storeN(iRegN src, memory4 mem)
7220 %{
7221 match(Set mem (StoreN mem src));
7222 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7223
7224 ins_cost(INSN_COST);
7225 format %{ "strw $src, $mem\t# compressed ptr" %}
7226
7227 ins_encode(aarch64_enc_strw(src, mem));
7228
7229 ins_pipe(istore_reg_mem);
7230 %}
7231
7232 instruct storeImmN0(immN0 zero, memory4 mem)
7233 %{
7234 match(Set mem (StoreN mem zero));
7235 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7236
7237 ins_cost(INSN_COST);
7238 format %{ "strw zr, $mem\t# compressed ptr" %}
7239
7240 ins_encode(aarch64_enc_strw0(mem));
7241
7242 ins_pipe(istore_mem);
7243 %}
7244
7245 // Store Float
7246 instruct storeF(vRegF src, memory4 mem)
7247 %{
7248 match(Set mem (StoreF mem src));
7249 predicate(!needs_releasing_store(n));
7250
7251 ins_cost(INSN_COST);
7252 format %{ "strs $src, $mem\t# float" %}
7253
7254 ins_encode( aarch64_enc_strs(src, mem) );
7255
7256 ins_pipe(pipe_class_memory);
7257 %}
7258
7259 // TODO
7260 // implement storeImmF0 and storeFImmPacked
7261
7262 // Store Double
7263 instruct storeD(vRegD src, memory8 mem)
7264 %{
7265 match(Set mem (StoreD mem src));
7266 predicate(!needs_releasing_store(n));
7267
7268 ins_cost(INSN_COST);
7269 format %{ "strd $src, $mem\t# double" %}
7270
7271 ins_encode( aarch64_enc_strd(src, mem) );
7272
7273 ins_pipe(pipe_class_memory);
7274 %}
7275
7276 // Store Compressed Klass Pointer
7277 instruct storeNKlass(iRegN src, memory4 mem)
7278 %{
7279 predicate(!needs_releasing_store(n));
7280 match(Set mem (StoreNKlass mem src));
7281
7282 ins_cost(INSN_COST);
7283 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7284
7285 ins_encode(aarch64_enc_strw(src, mem));
7286
7287 ins_pipe(istore_reg_mem);
7288 %}
7289
7290 // TODO
7291 // implement storeImmD0 and storeDImmPacked
7292
7293 // prefetch instructions
7294 // Must be safe to execute with invalid address (cannot fault).
7295
7296 instruct prefetchalloc( memory8 mem ) %{
7297 match(PrefetchAllocation mem);
7298
7299 ins_cost(INSN_COST);
7300 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7301
7302 ins_encode( aarch64_enc_prefetchw(mem) );
7303
7304 ins_pipe(iload_prefetch);
7305 %}
7306
7307 // ---------------- volatile loads and stores ----------------
7308
7309 // Load Byte (8 bit signed)
7310 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7311 %{
7312 match(Set dst (LoadB mem));
7313
7314 ins_cost(VOLATILE_REF_COST);
7315 format %{ "ldarsb $dst, $mem\t# byte" %}
7316
7317 ins_encode(aarch64_enc_ldarsb(dst, mem));
7318
7319 ins_pipe(pipe_serial);
7320 %}
7321
7322 // Load Byte (8 bit signed) into long
7323 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7324 %{
7325 match(Set dst (ConvI2L (LoadB mem)));
7326
7327 ins_cost(VOLATILE_REF_COST);
7328 format %{ "ldarsb $dst, $mem\t# byte" %}
7329
7330 ins_encode(aarch64_enc_ldarsb(dst, mem));
7331
7332 ins_pipe(pipe_serial);
7333 %}
7334
7335 // Load Byte (8 bit unsigned)
7336 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7337 %{
7338 match(Set dst (LoadUB mem));
7339
7340 ins_cost(VOLATILE_REF_COST);
7341 format %{ "ldarb $dst, $mem\t# byte" %}
7342
7343 ins_encode(aarch64_enc_ldarb(dst, mem));
7344
7345 ins_pipe(pipe_serial);
7346 %}
7347
7348 // Load Byte (8 bit unsigned) into long
7349 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7350 %{
7351 match(Set dst (ConvI2L (LoadUB mem)));
7352
7353 ins_cost(VOLATILE_REF_COST);
7354 format %{ "ldarb $dst, $mem\t# byte" %}
7355
7356 ins_encode(aarch64_enc_ldarb(dst, mem));
7357
7358 ins_pipe(pipe_serial);
7359 %}
7360
7361 // Load Short (16 bit signed)
7362 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7363 %{
7364 match(Set dst (LoadS mem));
7365
7366 ins_cost(VOLATILE_REF_COST);
7367 format %{ "ldarshw $dst, $mem\t# short" %}
7368
7369 ins_encode(aarch64_enc_ldarshw(dst, mem));
7370
7371 ins_pipe(pipe_serial);
7372 %}
7373
7374 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7375 %{
7376 match(Set dst (LoadUS mem));
7377
7378 ins_cost(VOLATILE_REF_COST);
7379 format %{ "ldarhw $dst, $mem\t# short" %}
7380
7381 ins_encode(aarch64_enc_ldarhw(dst, mem));
7382
7383 ins_pipe(pipe_serial);
7384 %}
7385
7386 // Load Short/Char (16 bit unsigned) into long
7387 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7388 %{
7389 match(Set dst (ConvI2L (LoadUS mem)));
7390
7391 ins_cost(VOLATILE_REF_COST);
7392 format %{ "ldarh $dst, $mem\t# short" %}
7393
7394 ins_encode(aarch64_enc_ldarh(dst, mem));
7395
7396 ins_pipe(pipe_serial);
7397 %}
7398
7399 // Load Short/Char (16 bit signed) into long
7400 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7401 %{
7402 match(Set dst (ConvI2L (LoadS mem)));
7403
7404 ins_cost(VOLATILE_REF_COST);
7405 format %{ "ldarh $dst, $mem\t# short" %}
7406
7407 ins_encode(aarch64_enc_ldarsh(dst, mem));
7408
7409 ins_pipe(pipe_serial);
7410 %}
7411
7412 // Load Integer (32 bit signed)
7413 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7414 %{
7415 match(Set dst (LoadI mem));
7416
7417 ins_cost(VOLATILE_REF_COST);
7418 format %{ "ldarw $dst, $mem\t# int" %}
7419
7420 ins_encode(aarch64_enc_ldarw(dst, mem));
7421
7422 ins_pipe(pipe_serial);
7423 %}
7424
7425 // Load Integer (32 bit unsigned) into long
7426 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7427 %{
7428 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7429
7430 ins_cost(VOLATILE_REF_COST);
7431 format %{ "ldarw $dst, $mem\t# int" %}
7432
7433 ins_encode(aarch64_enc_ldarw(dst, mem));
7434
7435 ins_pipe(pipe_serial);
7436 %}
7437
7438 // Load Long (64 bit signed)
7439 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7440 %{
7441 match(Set dst (LoadL mem));
7442
7443 ins_cost(VOLATILE_REF_COST);
7444 format %{ "ldar $dst, $mem\t# int" %}
7445
7446 ins_encode(aarch64_enc_ldar(dst, mem));
7447
7448 ins_pipe(pipe_serial);
7449 %}
7450
7451 // Load Pointer
7452 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7453 %{
7454 match(Set dst (LoadP mem));
7455 predicate(n->as_Load()->barrier_data() == 0);
7456
7457 ins_cost(VOLATILE_REF_COST);
7458 format %{ "ldar $dst, $mem\t# ptr" %}
7459
7460 ins_encode(aarch64_enc_ldar(dst, mem));
7461
7462 ins_pipe(pipe_serial);
7463 %}
7464
7465 // Load Compressed Pointer
7466 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7467 %{
7468 match(Set dst (LoadN mem));
7469 predicate(n->as_Load()->barrier_data() == 0);
7470
7471 ins_cost(VOLATILE_REF_COST);
7472 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7473
7474 ins_encode(aarch64_enc_ldarw(dst, mem));
7475
7476 ins_pipe(pipe_serial);
7477 %}
7478
7479 // Load Float
7480 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7481 %{
7482 match(Set dst (LoadF mem));
7483
7484 ins_cost(VOLATILE_REF_COST);
7485 format %{ "ldars $dst, $mem\t# float" %}
7486
7487 ins_encode( aarch64_enc_fldars(dst, mem) );
7488
7489 ins_pipe(pipe_serial);
7490 %}
7491
7492 // Load Double
7493 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7494 %{
7495 match(Set dst (LoadD mem));
7496
7497 ins_cost(VOLATILE_REF_COST);
7498 format %{ "ldard $dst, $mem\t# double" %}
7499
7500 ins_encode( aarch64_enc_fldard(dst, mem) );
7501
7502 ins_pipe(pipe_serial);
7503 %}
7504
7505 // Store Byte
7506 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7507 %{
7508 match(Set mem (StoreB mem src));
7509
7510 ins_cost(VOLATILE_REF_COST);
7511 format %{ "stlrb $src, $mem\t# byte" %}
7512
7513 ins_encode(aarch64_enc_stlrb(src, mem));
7514
7515 ins_pipe(pipe_class_memory);
7516 %}
7517
7518 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7519 %{
7520 match(Set mem (StoreB mem zero));
7521
7522 ins_cost(VOLATILE_REF_COST);
7523 format %{ "stlrb zr, $mem\t# byte" %}
7524
7525 ins_encode(aarch64_enc_stlrb0(mem));
7526
7527 ins_pipe(pipe_class_memory);
7528 %}
7529
7530 // Store Char/Short
7531 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7532 %{
7533 match(Set mem (StoreC mem src));
7534
7535 ins_cost(VOLATILE_REF_COST);
7536 format %{ "stlrh $src, $mem\t# short" %}
7537
7538 ins_encode(aarch64_enc_stlrh(src, mem));
7539
7540 ins_pipe(pipe_class_memory);
7541 %}
7542
7543 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7544 %{
7545 match(Set mem (StoreC mem zero));
7546
7547 ins_cost(VOLATILE_REF_COST);
7548 format %{ "stlrh zr, $mem\t# short" %}
7549
7550 ins_encode(aarch64_enc_stlrh0(mem));
7551
7552 ins_pipe(pipe_class_memory);
7553 %}
7554
7555 // Store Integer
7556
7557 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7558 %{
7559 match(Set mem(StoreI mem src));
7560
7561 ins_cost(VOLATILE_REF_COST);
7562 format %{ "stlrw $src, $mem\t# int" %}
7563
7564 ins_encode(aarch64_enc_stlrw(src, mem));
7565
7566 ins_pipe(pipe_class_memory);
7567 %}
7568
7569 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7570 %{
7571 match(Set mem(StoreI mem zero));
7572
7573 ins_cost(VOLATILE_REF_COST);
7574 format %{ "stlrw zr, $mem\t# int" %}
7575
7576 ins_encode(aarch64_enc_stlrw0(mem));
7577
7578 ins_pipe(pipe_class_memory);
7579 %}
7580
7581 // Store Long (64 bit signed)
7582 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7583 %{
7584 match(Set mem (StoreL mem src));
7585
7586 ins_cost(VOLATILE_REF_COST);
7587 format %{ "stlr $src, $mem\t# int" %}
7588
7589 ins_encode(aarch64_enc_stlr(src, mem));
7590
7591 ins_pipe(pipe_class_memory);
7592 %}
7593
7594 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7595 %{
7596 match(Set mem (StoreL mem zero));
7597
7598 ins_cost(VOLATILE_REF_COST);
7599 format %{ "stlr zr, $mem\t# int" %}
7600
7601 ins_encode(aarch64_enc_stlr0(mem));
7602
7603 ins_pipe(pipe_class_memory);
7604 %}
7605
7606 // Store Pointer
7607 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7608 %{
7609 match(Set mem (StoreP mem src));
7610 predicate(n->as_Store()->barrier_data() == 0);
7611
7612 ins_cost(VOLATILE_REF_COST);
7613 format %{ "stlr $src, $mem\t# ptr" %}
7614
7615 ins_encode(aarch64_enc_stlr(src, mem));
7616
7617 ins_pipe(pipe_class_memory);
7618 %}
7619
7620 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7621 %{
7622 match(Set mem (StoreP mem zero));
7623 predicate(n->as_Store()->barrier_data() == 0);
7624
7625 ins_cost(VOLATILE_REF_COST);
7626 format %{ "stlr zr, $mem\t# ptr" %}
7627
7628 ins_encode(aarch64_enc_stlr0(mem));
7629
7630 ins_pipe(pipe_class_memory);
7631 %}
7632
7633 // Store Compressed Pointer
7634 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7635 %{
7636 match(Set mem (StoreN mem src));
7637 predicate(n->as_Store()->barrier_data() == 0);
7638
7639 ins_cost(VOLATILE_REF_COST);
7640 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7641
7642 ins_encode(aarch64_enc_stlrw(src, mem));
7643
7644 ins_pipe(pipe_class_memory);
7645 %}
7646
7647 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7648 %{
7649 match(Set mem (StoreN mem zero));
7650 predicate(n->as_Store()->barrier_data() == 0);
7651
7652 ins_cost(VOLATILE_REF_COST);
7653 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7654
7655 ins_encode(aarch64_enc_stlrw0(mem));
7656
7657 ins_pipe(pipe_class_memory);
7658 %}
7659
7660 // Store Float
7661 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7662 %{
7663 match(Set mem (StoreF mem src));
7664
7665 ins_cost(VOLATILE_REF_COST);
7666 format %{ "stlrs $src, $mem\t# float" %}
7667
7668 ins_encode( aarch64_enc_fstlrs(src, mem) );
7669
7670 ins_pipe(pipe_class_memory);
7671 %}
7672
7673 // TODO
7674 // implement storeImmF0 and storeFImmPacked
7675
7676 // Store Double
7677 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7678 %{
7679 match(Set mem (StoreD mem src));
7680
7681 ins_cost(VOLATILE_REF_COST);
7682 format %{ "stlrd $src, $mem\t# double" %}
7683
7684 ins_encode( aarch64_enc_fstlrd(src, mem) );
7685
7686 ins_pipe(pipe_class_memory);
7687 %}
7688
7689 // ---------------- end of volatile loads and stores ----------------
7690
7691 instruct cacheWB(indirect addr)
7692 %{
7693 predicate(VM_Version::supports_data_cache_line_flush());
7694 match(CacheWB addr);
7695
7696 ins_cost(100);
7697 format %{"cache wb $addr" %}
7698 ins_encode %{
7699 assert($addr->index_position() < 0, "should be");
7700 assert($addr$$disp == 0, "should be");
7701 __ cache_wb(Address($addr$$base$$Register, 0));
7702 %}
7703 ins_pipe(pipe_slow); // XXX
7704 %}
7705
7706 instruct cacheWBPreSync()
7707 %{
7708 predicate(VM_Version::supports_data_cache_line_flush());
7709 match(CacheWBPreSync);
7710
7711 ins_cost(100);
7712 format %{"cache wb presync" %}
7713 ins_encode %{
7714 __ cache_wbsync(true);
7715 %}
7716 ins_pipe(pipe_slow); // XXX
7717 %}
7718
7719 instruct cacheWBPostSync()
7720 %{
7721 predicate(VM_Version::supports_data_cache_line_flush());
7722 match(CacheWBPostSync);
7723
7724 ins_cost(100);
7725 format %{"cache wb postsync" %}
7726 ins_encode %{
7727 __ cache_wbsync(false);
7728 %}
7729 ins_pipe(pipe_slow); // XXX
7730 %}
7731
7732 // ============================================================================
7733 // BSWAP Instructions
7734
7735 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7736 match(Set dst (ReverseBytesI src));
7737
7738 ins_cost(INSN_COST);
7739 format %{ "revw $dst, $src" %}
7740
7741 ins_encode %{
7742 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7743 %}
7744
7745 ins_pipe(ialu_reg);
7746 %}
7747
7748 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7749 match(Set dst (ReverseBytesL src));
7750
7751 ins_cost(INSN_COST);
7752 format %{ "rev $dst, $src" %}
7753
7754 ins_encode %{
7755 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7756 %}
7757
7758 ins_pipe(ialu_reg);
7759 %}
7760
7761 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7762 match(Set dst (ReverseBytesUS src));
7763
7764 ins_cost(INSN_COST);
7765 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7766
7767 ins_encode %{
7768 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7769 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7770 %}
7771
7772 ins_pipe(ialu_reg);
7773 %}
7774
7775 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7776 match(Set dst (ReverseBytesS src));
7777
7778 ins_cost(INSN_COST);
7779 format %{ "rev16w $dst, $src\n\t"
7780 "sbfmw $dst, $dst, #0, #15" %}
7781
7782 ins_encode %{
7783 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7784 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7785 %}
7786
7787 ins_pipe(ialu_reg);
7788 %}
7789
7790 // ============================================================================
7791 // Zero Count Instructions
7792
7793 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7794 match(Set dst (CountLeadingZerosI src));
7795
7796 ins_cost(INSN_COST);
7797 format %{ "clzw $dst, $src" %}
7798 ins_encode %{
7799 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7800 %}
7801
7802 ins_pipe(ialu_reg);
7803 %}
7804
7805 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7806 match(Set dst (CountLeadingZerosL src));
7807
7808 ins_cost(INSN_COST);
7809 format %{ "clz $dst, $src" %}
7810 ins_encode %{
7811 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7812 %}
7813
7814 ins_pipe(ialu_reg);
7815 %}
7816
7817 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7818 match(Set dst (CountTrailingZerosI src));
7819
7820 ins_cost(INSN_COST * 2);
7821 format %{ "rbitw $dst, $src\n\t"
7822 "clzw $dst, $dst" %}
7823 ins_encode %{
7824 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7825 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7826 %}
7827
7828 ins_pipe(ialu_reg);
7829 %}
7830
7831 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7832 match(Set dst (CountTrailingZerosL src));
7833
7834 ins_cost(INSN_COST * 2);
7835 format %{ "rbit $dst, $src\n\t"
7836 "clz $dst, $dst" %}
7837 ins_encode %{
7838 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7839 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7840 %}
7841
7842 ins_pipe(ialu_reg);
7843 %}
7844
7845 //---------- Population Count Instructions -------------------------------------
7846 //
7847
7848 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7849 match(Set dst (PopCountI src));
7850 effect(TEMP tmp);
7851 ins_cost(INSN_COST * 13);
7852
7853 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7854 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7855 "addv $tmp, $tmp\t# vector (8B)\n\t"
7856 "mov $dst, $tmp\t# vector (1D)" %}
7857 ins_encode %{
7858 __ fmovs($tmp$$FloatRegister, $src$$Register);
7859 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7860 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7861 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7862 %}
7863
7864 ins_pipe(pipe_class_default);
7865 %}
7866
7867 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7868 match(Set dst (PopCountI (LoadI mem)));
7869 effect(TEMP tmp);
7870 ins_cost(INSN_COST * 13);
7871
7872 format %{ "ldrs $tmp, $mem\n\t"
7873 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7874 "addv $tmp, $tmp\t# vector (8B)\n\t"
7875 "mov $dst, $tmp\t# vector (1D)" %}
7876 ins_encode %{
7877 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7878 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7879 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7880 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7881 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7882 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7883 %}
7884
7885 ins_pipe(pipe_class_default);
7886 %}
7887
7888 // Note: Long.bitCount(long) returns an int.
7889 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7890 match(Set dst (PopCountL src));
7891 effect(TEMP tmp);
7892 ins_cost(INSN_COST * 13);
7893
7894 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7895 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7896 "addv $tmp, $tmp\t# vector (8B)\n\t"
7897 "mov $dst, $tmp\t# vector (1D)" %}
7898 ins_encode %{
7899 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7900 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7901 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7902 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7903 %}
7904
7905 ins_pipe(pipe_class_default);
7906 %}
7907
7908 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7909 match(Set dst (PopCountL (LoadL mem)));
7910 effect(TEMP tmp);
7911 ins_cost(INSN_COST * 13);
7912
7913 format %{ "ldrd $tmp, $mem\n\t"
7914 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7915 "addv $tmp, $tmp\t# vector (8B)\n\t"
7916 "mov $dst, $tmp\t# vector (1D)" %}
7917 ins_encode %{
7918 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7919 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7921 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7922 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7923 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7924 %}
7925
7926 ins_pipe(pipe_class_default);
7927 %}
7928
7929 // ============================================================================
7930 // VerifyVectorAlignment Instruction
7931
7932 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7933 match(Set addr (VerifyVectorAlignment addr mask));
7934 effect(KILL cr);
7935 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7936 ins_encode %{
7937 Label Lskip;
7938 // check if masked bits of addr are zero
7939 __ tst($addr$$Register, $mask$$constant);
7940 __ br(Assembler::EQ, Lskip);
7941 __ stop("verify_vector_alignment found a misaligned vector memory access");
7942 __ bind(Lskip);
7943 %}
7944 ins_pipe(pipe_slow);
7945 %}
7946
7947 // ============================================================================
7948 // MemBar Instruction
7949
7950 instruct load_fence() %{
7951 match(LoadFence);
7952 ins_cost(VOLATILE_REF_COST);
7953
7954 format %{ "load_fence" %}
7955
7956 ins_encode %{
7957 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7958 %}
7959 ins_pipe(pipe_serial);
7960 %}
7961
7962 instruct unnecessary_membar_acquire() %{
7963 predicate(unnecessary_acquire(n));
7964 match(MemBarAcquire);
7965 ins_cost(0);
7966
7967 format %{ "membar_acquire (elided)" %}
7968
7969 ins_encode %{
7970 __ block_comment("membar_acquire (elided)");
7971 %}
7972
7973 ins_pipe(pipe_class_empty);
7974 %}
7975
7976 instruct membar_acquire() %{
7977 match(MemBarAcquire);
7978 ins_cost(VOLATILE_REF_COST);
7979
7980 format %{ "membar_acquire\n\t"
7981 "dmb ishld" %}
7982
7983 ins_encode %{
7984 __ block_comment("membar_acquire");
7985 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7986 %}
7987
7988 ins_pipe(pipe_serial);
7989 %}
7990
7991
7992 instruct membar_acquire_lock() %{
7993 match(MemBarAcquireLock);
7994 ins_cost(VOLATILE_REF_COST);
7995
7996 format %{ "membar_acquire_lock (elided)" %}
7997
7998 ins_encode %{
7999 __ block_comment("membar_acquire_lock (elided)");
8000 %}
8001
8002 ins_pipe(pipe_serial);
8003 %}
8004
8005 instruct store_fence() %{
8006 match(StoreFence);
8007 ins_cost(VOLATILE_REF_COST);
8008
8009 format %{ "store_fence" %}
8010
8011 ins_encode %{
8012 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8013 %}
8014 ins_pipe(pipe_serial);
8015 %}
8016
8017 instruct unnecessary_membar_release() %{
8018 predicate(unnecessary_release(n));
8019 match(MemBarRelease);
8020 ins_cost(0);
8021
8022 format %{ "membar_release (elided)" %}
8023
8024 ins_encode %{
8025 __ block_comment("membar_release (elided)");
8026 %}
8027 ins_pipe(pipe_serial);
8028 %}
8029
8030 instruct membar_release() %{
8031 match(MemBarRelease);
8032 ins_cost(VOLATILE_REF_COST);
8033
8034 format %{ "membar_release\n\t"
8035 "dmb ishst\n\tdmb ishld" %}
8036
8037 ins_encode %{
8038 __ block_comment("membar_release");
8039 // These will be merged if AlwaysMergeDMB is enabled.
8040 __ membar(Assembler::StoreStore);
8041 __ membar(Assembler::LoadStore);
8042 %}
8043 ins_pipe(pipe_serial);
8044 %}
8045
8046 instruct membar_storestore() %{
8047 match(MemBarStoreStore);
8048 match(StoreStoreFence);
8049 ins_cost(VOLATILE_REF_COST);
8050
8051 format %{ "MEMBAR-store-store" %}
8052
8053 ins_encode %{
8054 __ membar(Assembler::StoreStore);
8055 %}
8056 ins_pipe(pipe_serial);
8057 %}
8058
8059 instruct membar_release_lock() %{
8060 match(MemBarReleaseLock);
8061 ins_cost(VOLATILE_REF_COST);
8062
8063 format %{ "membar_release_lock (elided)" %}
8064
8065 ins_encode %{
8066 __ block_comment("membar_release_lock (elided)");
8067 %}
8068
8069 ins_pipe(pipe_serial);
8070 %}
8071
8072 instruct membar_storeload() %{
8073 match(MemBarStoreLoad);
8074 ins_cost(VOLATILE_REF_COST*100);
8075
8076 format %{ "MEMBAR-store-load\n\t"
8077 "dmb ish" %}
8078
8079 ins_encode %{
8080 __ block_comment("membar_storeload");
8081 __ membar(Assembler::StoreLoad);
8082 %}
8083
8084 ins_pipe(pipe_serial);
8085 %}
8086
8087 instruct unnecessary_membar_volatile() %{
8088 predicate(unnecessary_volatile(n));
8089 match(MemBarVolatile);
8090 ins_cost(0);
8091
8092 format %{ "membar_volatile (elided)" %}
8093
8094 ins_encode %{
8095 __ block_comment("membar_volatile (elided)");
8096 %}
8097
8098 ins_pipe(pipe_serial);
8099 %}
8100
8101 instruct membar_volatile() %{
8102 match(MemBarVolatile);
8103 ins_cost(VOLATILE_REF_COST*100);
8104
8105 format %{ "membar_volatile\n\t"
8106 "dmb ish"%}
8107
8108 ins_encode %{
8109 __ block_comment("membar_volatile");
8110 __ membar(Assembler::StoreLoad);
8111 %}
8112
8113 ins_pipe(pipe_serial);
8114 %}
8115
8116 instruct membar_full() %{
8117 match(MemBarFull);
8118 ins_cost(VOLATILE_REF_COST*100);
8119
8120 format %{ "membar_full\n\t"
8121 "dmb ish" %}
8122 ins_encode %{
8123 __ block_comment("membar_full");
8124 __ membar(Assembler::AnyAny);
8125 %}
8126
8127 ins_pipe(pipe_serial);
8128 %}
8129
8130 // ============================================================================
8131 // Cast/Convert Instructions
8132
8133 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8134 match(Set dst (CastX2P src));
8135
8136 ins_cost(INSN_COST);
8137 format %{ "mov $dst, $src\t# long -> ptr" %}
8138
8139 ins_encode %{
8140 if ($dst$$reg != $src$$reg) {
8141 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8142 }
8143 %}
8144
8145 ins_pipe(ialu_reg);
8146 %}
8147
8148 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8149 match(Set dst (CastI2N src));
8150
8151 ins_cost(INSN_COST);
8152 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8153
8154 ins_encode %{
8155 if ($dst$$reg != $src$$reg) {
8156 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8157 }
8158 %}
8159
8160 ins_pipe(ialu_reg);
8161 %}
8162
8163 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8164 match(Set dst (CastP2X src));
8165
8166 ins_cost(INSN_COST);
8167 format %{ "mov $dst, $src\t# ptr -> long" %}
8168
8169 ins_encode %{
8170 if ($dst$$reg != $src$$reg) {
8171 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8172 }
8173 %}
8174
8175 ins_pipe(ialu_reg);
8176 %}
8177
8178 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8179 match(Set dst (CastP2X src));
8180
8181 ins_cost(INSN_COST);
8182 format %{ "mov $dst, $src\t# ptr -> long" %}
8183
8184 ins_encode %{
8185 if ($dst$$reg != $src$$reg) {
8186 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8187 }
8188 %}
8189
8190 ins_pipe(ialu_reg);
8191 %}
8192
8193 // Convert oop into int for vectors alignment masking
8194 instruct convP2I(iRegINoSp dst, iRegP src) %{
8195 match(Set dst (ConvL2I (CastP2X src)));
8196
8197 ins_cost(INSN_COST);
8198 format %{ "movw $dst, $src\t# ptr -> int" %}
8199 ins_encode %{
8200 __ movw($dst$$Register, $src$$Register);
8201 %}
8202
8203 ins_pipe(ialu_reg);
8204 %}
8205
8206 // Convert compressed oop into int for vectors alignment masking
8207 // in case of 32bit oops (heap < 4Gb).
8208 instruct convN2I(iRegINoSp dst, iRegN src)
8209 %{
8210 predicate(CompressedOops::shift() == 0);
8211 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8212
8213 ins_cost(INSN_COST);
8214 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8215 ins_encode %{
8216 __ movw($dst$$Register, $src$$Register);
8217 %}
8218
8219 ins_pipe(ialu_reg);
8220 %}
8221
8222
8223 // Convert oop pointer into compressed form
8224 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8225 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8226 match(Set dst (EncodeP src));
8227 effect(KILL cr);
8228 ins_cost(INSN_COST * 3);
8229 format %{ "encode_heap_oop $dst, $src" %}
8230 ins_encode %{
8231 Register s = $src$$Register;
8232 Register d = $dst$$Register;
8233 __ encode_heap_oop(d, s);
8234 %}
8235 ins_pipe(ialu_reg);
8236 %}
8237
8238 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8239 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8240 match(Set dst (EncodeP src));
8241 ins_cost(INSN_COST * 3);
8242 format %{ "encode_heap_oop_not_null $dst, $src" %}
8243 ins_encode %{
8244 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8245 %}
8246 ins_pipe(ialu_reg);
8247 %}
8248
8249 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8250 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8251 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8252 match(Set dst (DecodeN src));
8253 ins_cost(INSN_COST * 3);
8254 format %{ "decode_heap_oop $dst, $src" %}
8255 ins_encode %{
8256 Register s = $src$$Register;
8257 Register d = $dst$$Register;
8258 __ decode_heap_oop(d, s);
8259 %}
8260 ins_pipe(ialu_reg);
8261 %}
8262
8263 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8264 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8265 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8266 match(Set dst (DecodeN src));
8267 ins_cost(INSN_COST * 3);
8268 format %{ "decode_heap_oop_not_null $dst, $src" %}
8269 ins_encode %{
8270 Register s = $src$$Register;
8271 Register d = $dst$$Register;
8272 __ decode_heap_oop_not_null(d, s);
8273 %}
8274 ins_pipe(ialu_reg);
8275 %}
8276
8277 // n.b. AArch64 implementations of encode_klass_not_null and
8278 // decode_klass_not_null do not modify the flags register so, unlike
8279 // Intel, we don't kill CR as a side effect here
8280
8281 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8282 match(Set dst (EncodePKlass src));
8283
8284 ins_cost(INSN_COST * 3);
8285 format %{ "encode_klass_not_null $dst,$src" %}
8286
8287 ins_encode %{
8288 Register src_reg = as_Register($src$$reg);
8289 Register dst_reg = as_Register($dst$$reg);
8290 __ encode_klass_not_null(dst_reg, src_reg);
8291 %}
8292
8293 ins_pipe(ialu_reg);
8294 %}
8295
8296 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8297 match(Set dst (DecodeNKlass src));
8298
8299 ins_cost(INSN_COST * 3);
8300 format %{ "decode_klass_not_null $dst,$src" %}
8301
8302 ins_encode %{
8303 Register src_reg = as_Register($src$$reg);
8304 Register dst_reg = as_Register($dst$$reg);
8305 if (dst_reg != src_reg) {
8306 __ decode_klass_not_null(dst_reg, src_reg);
8307 } else {
8308 __ decode_klass_not_null(dst_reg);
8309 }
8310 %}
8311
8312 ins_pipe(ialu_reg);
8313 %}
8314
8315 instruct checkCastPP(iRegPNoSp dst)
8316 %{
8317 match(Set dst (CheckCastPP dst));
8318
8319 size(0);
8320 format %{ "# checkcastPP of $dst" %}
8321 ins_encode(/* empty encoding */);
8322 ins_pipe(pipe_class_empty);
8323 %}
8324
8325 instruct castPP(iRegPNoSp dst)
8326 %{
8327 match(Set dst (CastPP dst));
8328
8329 size(0);
8330 format %{ "# castPP of $dst" %}
8331 ins_encode(/* empty encoding */);
8332 ins_pipe(pipe_class_empty);
8333 %}
8334
8335 instruct castII(iRegI dst)
8336 %{
8337 predicate(VerifyConstraintCasts == 0);
8338 match(Set dst (CastII dst));
8339
8340 size(0);
8341 format %{ "# castII of $dst" %}
8342 ins_encode(/* empty encoding */);
8343 ins_cost(0);
8344 ins_pipe(pipe_class_empty);
8345 %}
8346
8347 instruct castII_checked(iRegI dst, rFlagsReg cr)
8348 %{
8349 predicate(VerifyConstraintCasts > 0);
8350 match(Set dst (CastII dst));
8351 effect(KILL cr);
8352
8353 format %{ "# castII_checked of $dst" %}
8354 ins_encode %{
8355 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8356 %}
8357 ins_pipe(pipe_slow);
8358 %}
8359
8360 instruct castLL(iRegL dst)
8361 %{
8362 predicate(VerifyConstraintCasts == 0);
8363 match(Set dst (CastLL dst));
8364
8365 size(0);
8366 format %{ "# castLL of $dst" %}
8367 ins_encode(/* empty encoding */);
8368 ins_cost(0);
8369 ins_pipe(pipe_class_empty);
8370 %}
8371
8372 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8373 %{
8374 predicate(VerifyConstraintCasts > 0);
8375 match(Set dst (CastLL dst));
8376 effect(KILL cr);
8377
8378 format %{ "# castLL_checked of $dst" %}
8379 ins_encode %{
8380 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8381 %}
8382 ins_pipe(pipe_slow);
8383 %}
8384
8385 instruct castHH(vRegF dst)
8386 %{
8387 match(Set dst (CastHH dst));
8388 size(0);
8389 format %{ "# castHH of $dst" %}
8390 ins_encode(/* empty encoding */);
8391 ins_cost(0);
8392 ins_pipe(pipe_class_empty);
8393 %}
8394
8395 instruct castFF(vRegF dst)
8396 %{
8397 match(Set dst (CastFF dst));
8398
8399 size(0);
8400 format %{ "# castFF of $dst" %}
8401 ins_encode(/* empty encoding */);
8402 ins_cost(0);
8403 ins_pipe(pipe_class_empty);
8404 %}
8405
8406 instruct castDD(vRegD dst)
8407 %{
8408 match(Set dst (CastDD dst));
8409
8410 size(0);
8411 format %{ "# castDD of $dst" %}
8412 ins_encode(/* empty encoding */);
8413 ins_cost(0);
8414 ins_pipe(pipe_class_empty);
8415 %}
8416
8417 instruct castVV(vReg dst)
8418 %{
8419 match(Set dst (CastVV dst));
8420
8421 size(0);
8422 format %{ "# castVV of $dst" %}
8423 ins_encode(/* empty encoding */);
8424 ins_cost(0);
8425 ins_pipe(pipe_class_empty);
8426 %}
8427
8428 instruct castVVMask(pRegGov dst)
8429 %{
8430 match(Set dst (CastVV dst));
8431
8432 size(0);
8433 format %{ "# castVV of $dst" %}
8434 ins_encode(/* empty encoding */);
8435 ins_cost(0);
8436 ins_pipe(pipe_class_empty);
8437 %}
8438
8439 // Manifest a CmpU result in an integer register.
8440 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8441 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8442 %{
8443 match(Set dst (CmpU3 src1 src2));
8444 effect(KILL flags);
8445
8446 ins_cost(INSN_COST * 3);
8447 format %{
8448 "cmpw $src1, $src2\n\t"
8449 "csetw $dst, ne\n\t"
8450 "cnegw $dst, lo\t# CmpU3(reg)"
8451 %}
8452 ins_encode %{
8453 __ cmpw($src1$$Register, $src2$$Register);
8454 __ csetw($dst$$Register, Assembler::NE);
8455 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8456 %}
8457
8458 ins_pipe(pipe_class_default);
8459 %}
8460
8461 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8462 %{
8463 match(Set dst (CmpU3 src1 src2));
8464 effect(KILL flags);
8465
8466 ins_cost(INSN_COST * 3);
8467 format %{
8468 "subsw zr, $src1, $src2\n\t"
8469 "csetw $dst, ne\n\t"
8470 "cnegw $dst, lo\t# CmpU3(imm)"
8471 %}
8472 ins_encode %{
8473 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8474 __ csetw($dst$$Register, Assembler::NE);
8475 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8476 %}
8477
8478 ins_pipe(pipe_class_default);
8479 %}
8480
8481 // Manifest a CmpUL result in an integer register.
8482 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8483 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8484 %{
8485 match(Set dst (CmpUL3 src1 src2));
8486 effect(KILL flags);
8487
8488 ins_cost(INSN_COST * 3);
8489 format %{
8490 "cmp $src1, $src2\n\t"
8491 "csetw $dst, ne\n\t"
8492 "cnegw $dst, lo\t# CmpUL3(reg)"
8493 %}
8494 ins_encode %{
8495 __ cmp($src1$$Register, $src2$$Register);
8496 __ csetw($dst$$Register, Assembler::NE);
8497 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8498 %}
8499
8500 ins_pipe(pipe_class_default);
8501 %}
8502
8503 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8504 %{
8505 match(Set dst (CmpUL3 src1 src2));
8506 effect(KILL flags);
8507
8508 ins_cost(INSN_COST * 3);
8509 format %{
8510 "subs zr, $src1, $src2\n\t"
8511 "csetw $dst, ne\n\t"
8512 "cnegw $dst, lo\t# CmpUL3(imm)"
8513 %}
8514 ins_encode %{
8515 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8516 __ csetw($dst$$Register, Assembler::NE);
8517 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8518 %}
8519
8520 ins_pipe(pipe_class_default);
8521 %}
8522
8523 // Manifest a CmpL result in an integer register.
8524 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8525 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8526 %{
8527 match(Set dst (CmpL3 src1 src2));
8528 effect(KILL flags);
8529
8530 ins_cost(INSN_COST * 3);
8531 format %{
8532 "cmp $src1, $src2\n\t"
8533 "csetw $dst, ne\n\t"
8534 "cnegw $dst, lt\t# CmpL3(reg)"
8535 %}
8536 ins_encode %{
8537 __ cmp($src1$$Register, $src2$$Register);
8538 __ csetw($dst$$Register, Assembler::NE);
8539 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8540 %}
8541
8542 ins_pipe(pipe_class_default);
8543 %}
8544
8545 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8546 %{
8547 match(Set dst (CmpL3 src1 src2));
8548 effect(KILL flags);
8549
8550 ins_cost(INSN_COST * 3);
8551 format %{
8552 "subs zr, $src1, $src2\n\t"
8553 "csetw $dst, ne\n\t"
8554 "cnegw $dst, lt\t# CmpL3(imm)"
8555 %}
8556 ins_encode %{
8557 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8558 __ csetw($dst$$Register, Assembler::NE);
8559 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8560 %}
8561
8562 ins_pipe(pipe_class_default);
8563 %}
8564
8565 // ============================================================================
8566 // Conditional Move Instructions
8567
8568 // n.b. we have identical rules for both a signed compare op (cmpOp)
8569 // and an unsigned compare op (cmpOpU). it would be nice if we could
8570 // define an op class which merged both inputs and use it to type the
8571 // argument to a single rule. unfortunatelyt his fails because the
8572 // opclass does not live up to the COND_INTER interface of its
8573 // component operands. When the generic code tries to negate the
8574 // operand it ends up running the generci Machoper::negate method
8575 // which throws a ShouldNotHappen. So, we have to provide two flavours
8576 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8577
8578 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8579 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8580
8581 ins_cost(INSN_COST * 2);
8582 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8583
8584 ins_encode %{
8585 __ cselw(as_Register($dst$$reg),
8586 as_Register($src2$$reg),
8587 as_Register($src1$$reg),
8588 (Assembler::Condition)$cmp$$cmpcode);
8589 %}
8590
8591 ins_pipe(icond_reg_reg);
8592 %}
8593
8594 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8595 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8596
8597 ins_cost(INSN_COST * 2);
8598 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8599
8600 ins_encode %{
8601 __ cselw(as_Register($dst$$reg),
8602 as_Register($src2$$reg),
8603 as_Register($src1$$reg),
8604 (Assembler::Condition)$cmp$$cmpcode);
8605 %}
8606
8607 ins_pipe(icond_reg_reg);
8608 %}
8609
8610 // special cases where one arg is zero
8611
8612 // n.b. this is selected in preference to the rule above because it
8613 // avoids loading constant 0 into a source register
8614
8615 // TODO
8616 // we ought only to be able to cull one of these variants as the ideal
8617 // transforms ought always to order the zero consistently (to left/right?)
8618
8619 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8620 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8621
8622 ins_cost(INSN_COST * 2);
8623 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8624
8625 ins_encode %{
8626 __ cselw(as_Register($dst$$reg),
8627 as_Register($src$$reg),
8628 zr,
8629 (Assembler::Condition)$cmp$$cmpcode);
8630 %}
8631
8632 ins_pipe(icond_reg);
8633 %}
8634
8635 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8636 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8637
8638 ins_cost(INSN_COST * 2);
8639 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8640
8641 ins_encode %{
8642 __ cselw(as_Register($dst$$reg),
8643 as_Register($src$$reg),
8644 zr,
8645 (Assembler::Condition)$cmp$$cmpcode);
8646 %}
8647
8648 ins_pipe(icond_reg);
8649 %}
8650
8651 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8652 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8653
8654 ins_cost(INSN_COST * 2);
8655 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8656
8657 ins_encode %{
8658 __ cselw(as_Register($dst$$reg),
8659 zr,
8660 as_Register($src$$reg),
8661 (Assembler::Condition)$cmp$$cmpcode);
8662 %}
8663
8664 ins_pipe(icond_reg);
8665 %}
8666
8667 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8668 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8669
8670 ins_cost(INSN_COST * 2);
8671 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8672
8673 ins_encode %{
8674 __ cselw(as_Register($dst$$reg),
8675 zr,
8676 as_Register($src$$reg),
8677 (Assembler::Condition)$cmp$$cmpcode);
8678 %}
8679
8680 ins_pipe(icond_reg);
8681 %}
8682
8683 // special case for creating a boolean 0 or 1
8684
8685 // n.b. this is selected in preference to the rule above because it
8686 // avoids loading constants 0 and 1 into a source register
8687
8688 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8689 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8690
8691 ins_cost(INSN_COST * 2);
8692 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8693
8694 ins_encode %{
8695 // equivalently
8696 // cset(as_Register($dst$$reg),
8697 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8698 __ csincw(as_Register($dst$$reg),
8699 zr,
8700 zr,
8701 (Assembler::Condition)$cmp$$cmpcode);
8702 %}
8703
8704 ins_pipe(icond_none);
8705 %}
8706
8707 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8708 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8709
8710 ins_cost(INSN_COST * 2);
8711 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8712
8713 ins_encode %{
8714 // equivalently
8715 // cset(as_Register($dst$$reg),
8716 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8717 __ csincw(as_Register($dst$$reg),
8718 zr,
8719 zr,
8720 (Assembler::Condition)$cmp$$cmpcode);
8721 %}
8722
8723 ins_pipe(icond_none);
8724 %}
8725
8726 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8727 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8728
8729 ins_cost(INSN_COST * 2);
8730 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8731
8732 ins_encode %{
8733 __ csel(as_Register($dst$$reg),
8734 as_Register($src2$$reg),
8735 as_Register($src1$$reg),
8736 (Assembler::Condition)$cmp$$cmpcode);
8737 %}
8738
8739 ins_pipe(icond_reg_reg);
8740 %}
8741
8742 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8743 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8744
8745 ins_cost(INSN_COST * 2);
8746 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8747
8748 ins_encode %{
8749 __ csel(as_Register($dst$$reg),
8750 as_Register($src2$$reg),
8751 as_Register($src1$$reg),
8752 (Assembler::Condition)$cmp$$cmpcode);
8753 %}
8754
8755 ins_pipe(icond_reg_reg);
8756 %}
8757
8758 // special cases where one arg is zero
8759
8760 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8761 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8762
8763 ins_cost(INSN_COST * 2);
8764 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8765
8766 ins_encode %{
8767 __ csel(as_Register($dst$$reg),
8768 zr,
8769 as_Register($src$$reg),
8770 (Assembler::Condition)$cmp$$cmpcode);
8771 %}
8772
8773 ins_pipe(icond_reg);
8774 %}
8775
8776 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8777 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8778
8779 ins_cost(INSN_COST * 2);
8780 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8781
8782 ins_encode %{
8783 __ csel(as_Register($dst$$reg),
8784 zr,
8785 as_Register($src$$reg),
8786 (Assembler::Condition)$cmp$$cmpcode);
8787 %}
8788
8789 ins_pipe(icond_reg);
8790 %}
8791
8792 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8793 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8794
8795 ins_cost(INSN_COST * 2);
8796 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8797
8798 ins_encode %{
8799 __ csel(as_Register($dst$$reg),
8800 as_Register($src$$reg),
8801 zr,
8802 (Assembler::Condition)$cmp$$cmpcode);
8803 %}
8804
8805 ins_pipe(icond_reg);
8806 %}
8807
8808 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8809 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8810
8811 ins_cost(INSN_COST * 2);
8812 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8813
8814 ins_encode %{
8815 __ csel(as_Register($dst$$reg),
8816 as_Register($src$$reg),
8817 zr,
8818 (Assembler::Condition)$cmp$$cmpcode);
8819 %}
8820
8821 ins_pipe(icond_reg);
8822 %}
8823
8824 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8825 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8826
8827 ins_cost(INSN_COST * 2);
8828 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8829
8830 ins_encode %{
8831 __ csel(as_Register($dst$$reg),
8832 as_Register($src2$$reg),
8833 as_Register($src1$$reg),
8834 (Assembler::Condition)$cmp$$cmpcode);
8835 %}
8836
8837 ins_pipe(icond_reg_reg);
8838 %}
8839
8840 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8841 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8842
8843 ins_cost(INSN_COST * 2);
8844 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8845
8846 ins_encode %{
8847 __ csel(as_Register($dst$$reg),
8848 as_Register($src2$$reg),
8849 as_Register($src1$$reg),
8850 (Assembler::Condition)$cmp$$cmpcode);
8851 %}
8852
8853 ins_pipe(icond_reg_reg);
8854 %}
8855
8856 // special cases where one arg is zero
8857
8858 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8859 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8860
8861 ins_cost(INSN_COST * 2);
8862 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8863
8864 ins_encode %{
8865 __ csel(as_Register($dst$$reg),
8866 zr,
8867 as_Register($src$$reg),
8868 (Assembler::Condition)$cmp$$cmpcode);
8869 %}
8870
8871 ins_pipe(icond_reg);
8872 %}
8873
8874 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8875 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8876
8877 ins_cost(INSN_COST * 2);
8878 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8879
8880 ins_encode %{
8881 __ csel(as_Register($dst$$reg),
8882 zr,
8883 as_Register($src$$reg),
8884 (Assembler::Condition)$cmp$$cmpcode);
8885 %}
8886
8887 ins_pipe(icond_reg);
8888 %}
8889
8890 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8891 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8892
8893 ins_cost(INSN_COST * 2);
8894 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8895
8896 ins_encode %{
8897 __ csel(as_Register($dst$$reg),
8898 as_Register($src$$reg),
8899 zr,
8900 (Assembler::Condition)$cmp$$cmpcode);
8901 %}
8902
8903 ins_pipe(icond_reg);
8904 %}
8905
8906 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8907 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8908
8909 ins_cost(INSN_COST * 2);
8910 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8911
8912 ins_encode %{
8913 __ csel(as_Register($dst$$reg),
8914 as_Register($src$$reg),
8915 zr,
8916 (Assembler::Condition)$cmp$$cmpcode);
8917 %}
8918
8919 ins_pipe(icond_reg);
8920 %}
8921
8922 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8923 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8924
8925 ins_cost(INSN_COST * 2);
8926 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8927
8928 ins_encode %{
8929 __ cselw(as_Register($dst$$reg),
8930 as_Register($src2$$reg),
8931 as_Register($src1$$reg),
8932 (Assembler::Condition)$cmp$$cmpcode);
8933 %}
8934
8935 ins_pipe(icond_reg_reg);
8936 %}
8937
8938 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8939 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8940
8941 ins_cost(INSN_COST * 2);
8942 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8943
8944 ins_encode %{
8945 __ cselw(as_Register($dst$$reg),
8946 as_Register($src2$$reg),
8947 as_Register($src1$$reg),
8948 (Assembler::Condition)$cmp$$cmpcode);
8949 %}
8950
8951 ins_pipe(icond_reg_reg);
8952 %}
8953
8954 // special cases where one arg is zero
8955
8956 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8957 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8958
8959 ins_cost(INSN_COST * 2);
8960 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8961
8962 ins_encode %{
8963 __ cselw(as_Register($dst$$reg),
8964 zr,
8965 as_Register($src$$reg),
8966 (Assembler::Condition)$cmp$$cmpcode);
8967 %}
8968
8969 ins_pipe(icond_reg);
8970 %}
8971
8972 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8973 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8974
8975 ins_cost(INSN_COST * 2);
8976 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8977
8978 ins_encode %{
8979 __ cselw(as_Register($dst$$reg),
8980 zr,
8981 as_Register($src$$reg),
8982 (Assembler::Condition)$cmp$$cmpcode);
8983 %}
8984
8985 ins_pipe(icond_reg);
8986 %}
8987
8988 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8989 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8990
8991 ins_cost(INSN_COST * 2);
8992 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8993
8994 ins_encode %{
8995 __ cselw(as_Register($dst$$reg),
8996 as_Register($src$$reg),
8997 zr,
8998 (Assembler::Condition)$cmp$$cmpcode);
8999 %}
9000
9001 ins_pipe(icond_reg);
9002 %}
9003
9004 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9005 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9006
9007 ins_cost(INSN_COST * 2);
9008 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9009
9010 ins_encode %{
9011 __ cselw(as_Register($dst$$reg),
9012 as_Register($src$$reg),
9013 zr,
9014 (Assembler::Condition)$cmp$$cmpcode);
9015 %}
9016
9017 ins_pipe(icond_reg);
9018 %}
9019
9020 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9021 %{
9022 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9023
9024 ins_cost(INSN_COST * 3);
9025
9026 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9027 ins_encode %{
9028 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9029 __ fcsels(as_FloatRegister($dst$$reg),
9030 as_FloatRegister($src2$$reg),
9031 as_FloatRegister($src1$$reg),
9032 cond);
9033 %}
9034
9035 ins_pipe(fp_cond_reg_reg_s);
9036 %}
9037
9038 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9039 %{
9040 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9041
9042 ins_cost(INSN_COST * 3);
9043
9044 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9045 ins_encode %{
9046 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9047 __ fcsels(as_FloatRegister($dst$$reg),
9048 as_FloatRegister($src2$$reg),
9049 as_FloatRegister($src1$$reg),
9050 cond);
9051 %}
9052
9053 ins_pipe(fp_cond_reg_reg_s);
9054 %}
9055
9056 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9057 %{
9058 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9059
9060 ins_cost(INSN_COST * 3);
9061
9062 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9063 ins_encode %{
9064 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9065 __ fcseld(as_FloatRegister($dst$$reg),
9066 as_FloatRegister($src2$$reg),
9067 as_FloatRegister($src1$$reg),
9068 cond);
9069 %}
9070
9071 ins_pipe(fp_cond_reg_reg_d);
9072 %}
9073
9074 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9075 %{
9076 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9077
9078 ins_cost(INSN_COST * 3);
9079
9080 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9081 ins_encode %{
9082 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9083 __ fcseld(as_FloatRegister($dst$$reg),
9084 as_FloatRegister($src2$$reg),
9085 as_FloatRegister($src1$$reg),
9086 cond);
9087 %}
9088
9089 ins_pipe(fp_cond_reg_reg_d);
9090 %}
9091
9092 // ============================================================================
9093 // Arithmetic Instructions
9094 //
9095
9096 // Integer Addition
9097
9098 // TODO
9099 // these currently employ operations which do not set CR and hence are
9100 // not flagged as killing CR but we would like to isolate the cases
9101 // where we want to set flags from those where we don't. need to work
9102 // out how to do that.
9103
9104 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9105 match(Set dst (AddI src1 src2));
9106
9107 ins_cost(INSN_COST);
9108 format %{ "addw $dst, $src1, $src2" %}
9109
9110 ins_encode %{
9111 __ addw(as_Register($dst$$reg),
9112 as_Register($src1$$reg),
9113 as_Register($src2$$reg));
9114 %}
9115
9116 ins_pipe(ialu_reg_reg);
9117 %}
9118
9119 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9120 match(Set dst (AddI src1 src2));
9121
9122 ins_cost(INSN_COST);
9123 format %{ "addw $dst, $src1, $src2" %}
9124
9125 // use opcode to indicate that this is an add not a sub
9126 opcode(0x0);
9127
9128 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9129
9130 ins_pipe(ialu_reg_imm);
9131 %}
9132
9133 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9134 match(Set dst (AddI (ConvL2I src1) src2));
9135
9136 ins_cost(INSN_COST);
9137 format %{ "addw $dst, $src1, $src2" %}
9138
9139 // use opcode to indicate that this is an add not a sub
9140 opcode(0x0);
9141
9142 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9143
9144 ins_pipe(ialu_reg_imm);
9145 %}
9146
9147 // Pointer Addition
9148 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9149 match(Set dst (AddP src1 src2));
9150
9151 ins_cost(INSN_COST);
9152 format %{ "add $dst, $src1, $src2\t# ptr" %}
9153
9154 ins_encode %{
9155 __ add(as_Register($dst$$reg),
9156 as_Register($src1$$reg),
9157 as_Register($src2$$reg));
9158 %}
9159
9160 ins_pipe(ialu_reg_reg);
9161 %}
9162
9163 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9164 match(Set dst (AddP src1 (ConvI2L src2)));
9165
9166 ins_cost(1.9 * INSN_COST);
9167 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9168
9169 ins_encode %{
9170 __ add(as_Register($dst$$reg),
9171 as_Register($src1$$reg),
9172 as_Register($src2$$reg), ext::sxtw);
9173 %}
9174
9175 ins_pipe(ialu_reg_reg);
9176 %}
9177
9178 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9179 match(Set dst (AddP src1 (LShiftL src2 scale)));
9180
9181 ins_cost(1.9 * INSN_COST);
9182 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9183
9184 ins_encode %{
9185 __ lea(as_Register($dst$$reg),
9186 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9187 Address::lsl($scale$$constant)));
9188 %}
9189
9190 ins_pipe(ialu_reg_reg_shift);
9191 %}
9192
9193 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9194 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9195
9196 ins_cost(1.9 * INSN_COST);
9197 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9198
9199 ins_encode %{
9200 __ lea(as_Register($dst$$reg),
9201 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9202 Address::sxtw($scale$$constant)));
9203 %}
9204
9205 ins_pipe(ialu_reg_reg_shift);
9206 %}
9207
9208 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9209 match(Set dst (LShiftL (ConvI2L src) scale));
9210
9211 ins_cost(INSN_COST);
9212 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9213
9214 ins_encode %{
9215 __ sbfiz(as_Register($dst$$reg),
9216 as_Register($src$$reg),
9217 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9218 %}
9219
9220 ins_pipe(ialu_reg_shift);
9221 %}
9222
9223 // Pointer Immediate Addition
9224 // n.b. this needs to be more expensive than using an indirect memory
9225 // operand
9226 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9227 match(Set dst (AddP src1 src2));
9228
9229 ins_cost(INSN_COST);
9230 format %{ "add $dst, $src1, $src2\t# ptr" %}
9231
9232 // use opcode to indicate that this is an add not a sub
9233 opcode(0x0);
9234
9235 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9236
9237 ins_pipe(ialu_reg_imm);
9238 %}
9239
9240 // Long Addition
9241 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9242
9243 match(Set dst (AddL src1 src2));
9244
9245 ins_cost(INSN_COST);
9246 format %{ "add $dst, $src1, $src2" %}
9247
9248 ins_encode %{
9249 __ add(as_Register($dst$$reg),
9250 as_Register($src1$$reg),
9251 as_Register($src2$$reg));
9252 %}
9253
9254 ins_pipe(ialu_reg_reg);
9255 %}
9256
9257 // No constant pool entries requiredLong Immediate Addition.
9258 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9259 match(Set dst (AddL src1 src2));
9260
9261 ins_cost(INSN_COST);
9262 format %{ "add $dst, $src1, $src2" %}
9263
9264 // use opcode to indicate that this is an add not a sub
9265 opcode(0x0);
9266
9267 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9268
9269 ins_pipe(ialu_reg_imm);
9270 %}
9271
9272 // Integer Subtraction
9273 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9274 match(Set dst (SubI src1 src2));
9275
9276 ins_cost(INSN_COST);
9277 format %{ "subw $dst, $src1, $src2" %}
9278
9279 ins_encode %{
9280 __ subw(as_Register($dst$$reg),
9281 as_Register($src1$$reg),
9282 as_Register($src2$$reg));
9283 %}
9284
9285 ins_pipe(ialu_reg_reg);
9286 %}
9287
9288 // Immediate Subtraction
9289 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9290 match(Set dst (SubI src1 src2));
9291
9292 ins_cost(INSN_COST);
9293 format %{ "subw $dst, $src1, $src2" %}
9294
9295 // use opcode to indicate that this is a sub not an add
9296 opcode(0x1);
9297
9298 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9299
9300 ins_pipe(ialu_reg_imm);
9301 %}
9302
9303 // Long Subtraction
9304 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9305
9306 match(Set dst (SubL src1 src2));
9307
9308 ins_cost(INSN_COST);
9309 format %{ "sub $dst, $src1, $src2" %}
9310
9311 ins_encode %{
9312 __ sub(as_Register($dst$$reg),
9313 as_Register($src1$$reg),
9314 as_Register($src2$$reg));
9315 %}
9316
9317 ins_pipe(ialu_reg_reg);
9318 %}
9319
9320 // No constant pool entries requiredLong Immediate Subtraction.
9321 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9322 match(Set dst (SubL src1 src2));
9323
9324 ins_cost(INSN_COST);
9325 format %{ "sub$dst, $src1, $src2" %}
9326
9327 // use opcode to indicate that this is a sub not an add
9328 opcode(0x1);
9329
9330 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9331
9332 ins_pipe(ialu_reg_imm);
9333 %}
9334
9335 // Integer Negation (special case for sub)
9336
9337 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9338 match(Set dst (SubI zero src));
9339
9340 ins_cost(INSN_COST);
9341 format %{ "negw $dst, $src\t# int" %}
9342
9343 ins_encode %{
9344 __ negw(as_Register($dst$$reg),
9345 as_Register($src$$reg));
9346 %}
9347
9348 ins_pipe(ialu_reg);
9349 %}
9350
9351 // Long Negation
9352
9353 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9354 match(Set dst (SubL zero src));
9355
9356 ins_cost(INSN_COST);
9357 format %{ "neg $dst, $src\t# long" %}
9358
9359 ins_encode %{
9360 __ neg(as_Register($dst$$reg),
9361 as_Register($src$$reg));
9362 %}
9363
9364 ins_pipe(ialu_reg);
9365 %}
9366
9367 // Integer Multiply
9368
9369 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9370 match(Set dst (MulI src1 src2));
9371
9372 ins_cost(INSN_COST * 3);
9373 format %{ "mulw $dst, $src1, $src2" %}
9374
9375 ins_encode %{
9376 __ mulw(as_Register($dst$$reg),
9377 as_Register($src1$$reg),
9378 as_Register($src2$$reg));
9379 %}
9380
9381 ins_pipe(imul_reg_reg);
9382 %}
9383
9384 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9385 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9386
9387 ins_cost(INSN_COST * 3);
9388 format %{ "smull $dst, $src1, $src2" %}
9389
9390 ins_encode %{
9391 __ smull(as_Register($dst$$reg),
9392 as_Register($src1$$reg),
9393 as_Register($src2$$reg));
9394 %}
9395
9396 ins_pipe(imul_reg_reg);
9397 %}
9398
9399 // Long Multiply
9400
9401 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9402 match(Set dst (MulL src1 src2));
9403
9404 ins_cost(INSN_COST * 5);
9405 format %{ "mul $dst, $src1, $src2" %}
9406
9407 ins_encode %{
9408 __ mul(as_Register($dst$$reg),
9409 as_Register($src1$$reg),
9410 as_Register($src2$$reg));
9411 %}
9412
9413 ins_pipe(lmul_reg_reg);
9414 %}
9415
9416 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9417 %{
9418 match(Set dst (MulHiL src1 src2));
9419
9420 ins_cost(INSN_COST * 7);
9421 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9422
9423 ins_encode %{
9424 __ smulh(as_Register($dst$$reg),
9425 as_Register($src1$$reg),
9426 as_Register($src2$$reg));
9427 %}
9428
9429 ins_pipe(lmul_reg_reg);
9430 %}
9431
9432 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9433 %{
9434 match(Set dst (UMulHiL src1 src2));
9435
9436 ins_cost(INSN_COST * 7);
9437 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9438
9439 ins_encode %{
9440 __ umulh(as_Register($dst$$reg),
9441 as_Register($src1$$reg),
9442 as_Register($src2$$reg));
9443 %}
9444
9445 ins_pipe(lmul_reg_reg);
9446 %}
9447
9448 // Combined Integer Multiply & Add/Sub
9449
9450 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9451 match(Set dst (AddI src3 (MulI src1 src2)));
9452
9453 ins_cost(INSN_COST * 3);
9454 format %{ "madd $dst, $src1, $src2, $src3" %}
9455
9456 ins_encode %{
9457 __ maddw(as_Register($dst$$reg),
9458 as_Register($src1$$reg),
9459 as_Register($src2$$reg),
9460 as_Register($src3$$reg));
9461 %}
9462
9463 ins_pipe(imac_reg_reg);
9464 %}
9465
9466 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9467 match(Set dst (SubI src3 (MulI src1 src2)));
9468
9469 ins_cost(INSN_COST * 3);
9470 format %{ "msub $dst, $src1, $src2, $src3" %}
9471
9472 ins_encode %{
9473 __ msubw(as_Register($dst$$reg),
9474 as_Register($src1$$reg),
9475 as_Register($src2$$reg),
9476 as_Register($src3$$reg));
9477 %}
9478
9479 ins_pipe(imac_reg_reg);
9480 %}
9481
9482 // Combined Integer Multiply & Neg
9483
9484 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9485 match(Set dst (MulI (SubI zero src1) src2));
9486
9487 ins_cost(INSN_COST * 3);
9488 format %{ "mneg $dst, $src1, $src2" %}
9489
9490 ins_encode %{
9491 __ mnegw(as_Register($dst$$reg),
9492 as_Register($src1$$reg),
9493 as_Register($src2$$reg));
9494 %}
9495
9496 ins_pipe(imac_reg_reg);
9497 %}
9498
9499 // Combined Long Multiply & Add/Sub
9500
9501 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9502 match(Set dst (AddL src3 (MulL src1 src2)));
9503
9504 ins_cost(INSN_COST * 5);
9505 format %{ "madd $dst, $src1, $src2, $src3" %}
9506
9507 ins_encode %{
9508 __ madd(as_Register($dst$$reg),
9509 as_Register($src1$$reg),
9510 as_Register($src2$$reg),
9511 as_Register($src3$$reg));
9512 %}
9513
9514 ins_pipe(lmac_reg_reg);
9515 %}
9516
9517 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9518 match(Set dst (SubL src3 (MulL src1 src2)));
9519
9520 ins_cost(INSN_COST * 5);
9521 format %{ "msub $dst, $src1, $src2, $src3" %}
9522
9523 ins_encode %{
9524 __ msub(as_Register($dst$$reg),
9525 as_Register($src1$$reg),
9526 as_Register($src2$$reg),
9527 as_Register($src3$$reg));
9528 %}
9529
9530 ins_pipe(lmac_reg_reg);
9531 %}
9532
9533 // Combined Long Multiply & Neg
9534
9535 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9536 match(Set dst (MulL (SubL zero src1) src2));
9537
9538 ins_cost(INSN_COST * 5);
9539 format %{ "mneg $dst, $src1, $src2" %}
9540
9541 ins_encode %{
9542 __ mneg(as_Register($dst$$reg),
9543 as_Register($src1$$reg),
9544 as_Register($src2$$reg));
9545 %}
9546
9547 ins_pipe(lmac_reg_reg);
9548 %}
9549
9550 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9551
9552 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9553 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9554
9555 ins_cost(INSN_COST * 3);
9556 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9557
9558 ins_encode %{
9559 __ smaddl(as_Register($dst$$reg),
9560 as_Register($src1$$reg),
9561 as_Register($src2$$reg),
9562 as_Register($src3$$reg));
9563 %}
9564
9565 ins_pipe(imac_reg_reg);
9566 %}
9567
9568 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9569 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9570
9571 ins_cost(INSN_COST * 3);
9572 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9573
9574 ins_encode %{
9575 __ smsubl(as_Register($dst$$reg),
9576 as_Register($src1$$reg),
9577 as_Register($src2$$reg),
9578 as_Register($src3$$reg));
9579 %}
9580
9581 ins_pipe(imac_reg_reg);
9582 %}
9583
9584 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9585 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9586
9587 ins_cost(INSN_COST * 3);
9588 format %{ "smnegl $dst, $src1, $src2" %}
9589
9590 ins_encode %{
9591 __ smnegl(as_Register($dst$$reg),
9592 as_Register($src1$$reg),
9593 as_Register($src2$$reg));
9594 %}
9595
9596 ins_pipe(imac_reg_reg);
9597 %}
9598
9599 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9600
9601 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9602 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9603
9604 ins_cost(INSN_COST * 5);
9605 format %{ "mulw rscratch1, $src1, $src2\n\t"
9606 "maddw $dst, $src3, $src4, rscratch1" %}
9607
9608 ins_encode %{
9609 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9610 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9611
9612 ins_pipe(imac_reg_reg);
9613 %}
9614
9615 // Integer Divide
9616
9617 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9618 match(Set dst (DivI src1 src2));
9619
9620 ins_cost(INSN_COST * 19);
9621 format %{ "sdivw $dst, $src1, $src2" %}
9622
9623 ins_encode(aarch64_enc_divw(dst, src1, src2));
9624 ins_pipe(idiv_reg_reg);
9625 %}
9626
9627 // Long Divide
9628
9629 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9630 match(Set dst (DivL src1 src2));
9631
9632 ins_cost(INSN_COST * 35);
9633 format %{ "sdiv $dst, $src1, $src2" %}
9634
9635 ins_encode(aarch64_enc_div(dst, src1, src2));
9636 ins_pipe(ldiv_reg_reg);
9637 %}
9638
9639 // Integer Remainder
9640
9641 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9642 match(Set dst (ModI src1 src2));
9643
9644 ins_cost(INSN_COST * 22);
9645 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9646 "msubw $dst, rscratch1, $src2, $src1" %}
9647
9648 ins_encode(aarch64_enc_modw(dst, src1, src2));
9649 ins_pipe(idiv_reg_reg);
9650 %}
9651
9652 // Long Remainder
9653
9654 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9655 match(Set dst (ModL src1 src2));
9656
9657 ins_cost(INSN_COST * 38);
9658 format %{ "sdiv rscratch1, $src1, $src2\n"
9659 "msub $dst, rscratch1, $src2, $src1" %}
9660
9661 ins_encode(aarch64_enc_mod(dst, src1, src2));
9662 ins_pipe(ldiv_reg_reg);
9663 %}
9664
9665 // Unsigned Integer Divide
9666
9667 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9668 match(Set dst (UDivI src1 src2));
9669
9670 ins_cost(INSN_COST * 19);
9671 format %{ "udivw $dst, $src1, $src2" %}
9672
9673 ins_encode %{
9674 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9675 %}
9676
9677 ins_pipe(idiv_reg_reg);
9678 %}
9679
9680 // Unsigned Long Divide
9681
9682 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9683 match(Set dst (UDivL src1 src2));
9684
9685 ins_cost(INSN_COST * 35);
9686 format %{ "udiv $dst, $src1, $src2" %}
9687
9688 ins_encode %{
9689 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9690 %}
9691
9692 ins_pipe(ldiv_reg_reg);
9693 %}
9694
9695 // Unsigned Integer Remainder
9696
9697 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9698 match(Set dst (UModI src1 src2));
9699
9700 ins_cost(INSN_COST * 22);
9701 format %{ "udivw rscratch1, $src1, $src2\n\t"
9702 "msubw $dst, rscratch1, $src2, $src1" %}
9703
9704 ins_encode %{
9705 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9706 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9707 %}
9708
9709 ins_pipe(idiv_reg_reg);
9710 %}
9711
9712 // Unsigned Long Remainder
9713
9714 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9715 match(Set dst (UModL src1 src2));
9716
9717 ins_cost(INSN_COST * 38);
9718 format %{ "udiv rscratch1, $src1, $src2\n"
9719 "msub $dst, rscratch1, $src2, $src1" %}
9720
9721 ins_encode %{
9722 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9723 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9724 %}
9725
9726 ins_pipe(ldiv_reg_reg);
9727 %}
9728
9729 // Integer Shifts
9730
9731 // Shift Left Register
9732 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9733 match(Set dst (LShiftI src1 src2));
9734
9735 ins_cost(INSN_COST * 2);
9736 format %{ "lslvw $dst, $src1, $src2" %}
9737
9738 ins_encode %{
9739 __ lslvw(as_Register($dst$$reg),
9740 as_Register($src1$$reg),
9741 as_Register($src2$$reg));
9742 %}
9743
9744 ins_pipe(ialu_reg_reg_vshift);
9745 %}
9746
9747 // Shift Left Immediate
9748 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9749 match(Set dst (LShiftI src1 src2));
9750
9751 ins_cost(INSN_COST);
9752 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9753
9754 ins_encode %{
9755 __ lslw(as_Register($dst$$reg),
9756 as_Register($src1$$reg),
9757 $src2$$constant & 0x1f);
9758 %}
9759
9760 ins_pipe(ialu_reg_shift);
9761 %}
9762
9763 // Shift Right Logical Register
9764 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9765 match(Set dst (URShiftI src1 src2));
9766
9767 ins_cost(INSN_COST * 2);
9768 format %{ "lsrvw $dst, $src1, $src2" %}
9769
9770 ins_encode %{
9771 __ lsrvw(as_Register($dst$$reg),
9772 as_Register($src1$$reg),
9773 as_Register($src2$$reg));
9774 %}
9775
9776 ins_pipe(ialu_reg_reg_vshift);
9777 %}
9778
9779 // Shift Right Logical Immediate
9780 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9781 match(Set dst (URShiftI src1 src2));
9782
9783 ins_cost(INSN_COST);
9784 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9785
9786 ins_encode %{
9787 __ lsrw(as_Register($dst$$reg),
9788 as_Register($src1$$reg),
9789 $src2$$constant & 0x1f);
9790 %}
9791
9792 ins_pipe(ialu_reg_shift);
9793 %}
9794
9795 // Shift Right Arithmetic Register
9796 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9797 match(Set dst (RShiftI src1 src2));
9798
9799 ins_cost(INSN_COST * 2);
9800 format %{ "asrvw $dst, $src1, $src2" %}
9801
9802 ins_encode %{
9803 __ asrvw(as_Register($dst$$reg),
9804 as_Register($src1$$reg),
9805 as_Register($src2$$reg));
9806 %}
9807
9808 ins_pipe(ialu_reg_reg_vshift);
9809 %}
9810
9811 // Shift Right Arithmetic Immediate
9812 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9813 match(Set dst (RShiftI src1 src2));
9814
9815 ins_cost(INSN_COST);
9816 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9817
9818 ins_encode %{
9819 __ asrw(as_Register($dst$$reg),
9820 as_Register($src1$$reg),
9821 $src2$$constant & 0x1f);
9822 %}
9823
9824 ins_pipe(ialu_reg_shift);
9825 %}
9826
9827 // Combined Int Mask and Right Shift (using UBFM)
9828 // TODO
9829
9830 // Long Shifts
9831
9832 // Shift Left Register
9833 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9834 match(Set dst (LShiftL src1 src2));
9835
9836 ins_cost(INSN_COST * 2);
9837 format %{ "lslv $dst, $src1, $src2" %}
9838
9839 ins_encode %{
9840 __ lslv(as_Register($dst$$reg),
9841 as_Register($src1$$reg),
9842 as_Register($src2$$reg));
9843 %}
9844
9845 ins_pipe(ialu_reg_reg_vshift);
9846 %}
9847
9848 // Shift Left Immediate
9849 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9850 match(Set dst (LShiftL src1 src2));
9851
9852 ins_cost(INSN_COST);
9853 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9854
9855 ins_encode %{
9856 __ lsl(as_Register($dst$$reg),
9857 as_Register($src1$$reg),
9858 $src2$$constant & 0x3f);
9859 %}
9860
9861 ins_pipe(ialu_reg_shift);
9862 %}
9863
9864 // Shift Right Logical Register
9865 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9866 match(Set dst (URShiftL src1 src2));
9867
9868 ins_cost(INSN_COST * 2);
9869 format %{ "lsrv $dst, $src1, $src2" %}
9870
9871 ins_encode %{
9872 __ lsrv(as_Register($dst$$reg),
9873 as_Register($src1$$reg),
9874 as_Register($src2$$reg));
9875 %}
9876
9877 ins_pipe(ialu_reg_reg_vshift);
9878 %}
9879
9880 // Shift Right Logical Immediate
9881 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9882 match(Set dst (URShiftL src1 src2));
9883
9884 ins_cost(INSN_COST);
9885 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9886
9887 ins_encode %{
9888 __ lsr(as_Register($dst$$reg),
9889 as_Register($src1$$reg),
9890 $src2$$constant & 0x3f);
9891 %}
9892
9893 ins_pipe(ialu_reg_shift);
9894 %}
9895
9896 // A special-case pattern for card table stores.
9897 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9898 match(Set dst (URShiftL (CastP2X src1) src2));
9899
9900 ins_cost(INSN_COST);
9901 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9902
9903 ins_encode %{
9904 __ lsr(as_Register($dst$$reg),
9905 as_Register($src1$$reg),
9906 $src2$$constant & 0x3f);
9907 %}
9908
9909 ins_pipe(ialu_reg_shift);
9910 %}
9911
9912 // Shift Right Arithmetic Register
9913 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9914 match(Set dst (RShiftL src1 src2));
9915
9916 ins_cost(INSN_COST * 2);
9917 format %{ "asrv $dst, $src1, $src2" %}
9918
9919 ins_encode %{
9920 __ asrv(as_Register($dst$$reg),
9921 as_Register($src1$$reg),
9922 as_Register($src2$$reg));
9923 %}
9924
9925 ins_pipe(ialu_reg_reg_vshift);
9926 %}
9927
9928 // Shift Right Arithmetic Immediate
9929 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9930 match(Set dst (RShiftL src1 src2));
9931
9932 ins_cost(INSN_COST);
9933 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9934
9935 ins_encode %{
9936 __ asr(as_Register($dst$$reg),
9937 as_Register($src1$$reg),
9938 $src2$$constant & 0x3f);
9939 %}
9940
9941 ins_pipe(ialu_reg_shift);
9942 %}
9943
9944 // BEGIN This section of the file is automatically generated. Do not edit --------------
9945 // This section is generated from aarch64_ad.m4
9946
9947 // This pattern is automatically generated from aarch64_ad.m4
9948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9949 instruct regL_not_reg(iRegLNoSp dst,
9950 iRegL src1, immL_M1 m1,
9951 rFlagsReg cr) %{
9952 match(Set dst (XorL src1 m1));
9953 ins_cost(INSN_COST);
9954 format %{ "eon $dst, $src1, zr" %}
9955
9956 ins_encode %{
9957 __ eon(as_Register($dst$$reg),
9958 as_Register($src1$$reg),
9959 zr,
9960 Assembler::LSL, 0);
9961 %}
9962
9963 ins_pipe(ialu_reg);
9964 %}
9965
9966 // This pattern is automatically generated from aarch64_ad.m4
9967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9968 instruct regI_not_reg(iRegINoSp dst,
9969 iRegIorL2I src1, immI_M1 m1,
9970 rFlagsReg cr) %{
9971 match(Set dst (XorI src1 m1));
9972 ins_cost(INSN_COST);
9973 format %{ "eonw $dst, $src1, zr" %}
9974
9975 ins_encode %{
9976 __ eonw(as_Register($dst$$reg),
9977 as_Register($src1$$reg),
9978 zr,
9979 Assembler::LSL, 0);
9980 %}
9981
9982 ins_pipe(ialu_reg);
9983 %}
9984
9985 // This pattern is automatically generated from aarch64_ad.m4
9986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9987 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9988 immI0 zero, iRegIorL2I src1, immI src2) %{
9989 match(Set dst (SubI zero (URShiftI src1 src2)));
9990
9991 ins_cost(1.9 * INSN_COST);
9992 format %{ "negw $dst, $src1, LSR $src2" %}
9993
9994 ins_encode %{
9995 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9996 Assembler::LSR, $src2$$constant & 0x1f);
9997 %}
9998
9999 ins_pipe(ialu_reg_shift);
10000 %}
10001
10002 // This pattern is automatically generated from aarch64_ad.m4
10003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10004 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10005 immI0 zero, iRegIorL2I src1, immI src2) %{
10006 match(Set dst (SubI zero (RShiftI src1 src2)));
10007
10008 ins_cost(1.9 * INSN_COST);
10009 format %{ "negw $dst, $src1, ASR $src2" %}
10010
10011 ins_encode %{
10012 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10013 Assembler::ASR, $src2$$constant & 0x1f);
10014 %}
10015
10016 ins_pipe(ialu_reg_shift);
10017 %}
10018
10019 // This pattern is automatically generated from aarch64_ad.m4
10020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10021 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10022 immI0 zero, iRegIorL2I src1, immI src2) %{
10023 match(Set dst (SubI zero (LShiftI src1 src2)));
10024
10025 ins_cost(1.9 * INSN_COST);
10026 format %{ "negw $dst, $src1, LSL $src2" %}
10027
10028 ins_encode %{
10029 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10030 Assembler::LSL, $src2$$constant & 0x1f);
10031 %}
10032
10033 ins_pipe(ialu_reg_shift);
10034 %}
10035
10036 // This pattern is automatically generated from aarch64_ad.m4
10037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10038 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10039 immL0 zero, iRegL src1, immI src2) %{
10040 match(Set dst (SubL zero (URShiftL src1 src2)));
10041
10042 ins_cost(1.9 * INSN_COST);
10043 format %{ "neg $dst, $src1, LSR $src2" %}
10044
10045 ins_encode %{
10046 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10047 Assembler::LSR, $src2$$constant & 0x3f);
10048 %}
10049
10050 ins_pipe(ialu_reg_shift);
10051 %}
10052
10053 // This pattern is automatically generated from aarch64_ad.m4
10054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10055 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10056 immL0 zero, iRegL src1, immI src2) %{
10057 match(Set dst (SubL zero (RShiftL src1 src2)));
10058
10059 ins_cost(1.9 * INSN_COST);
10060 format %{ "neg $dst, $src1, ASR $src2" %}
10061
10062 ins_encode %{
10063 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10064 Assembler::ASR, $src2$$constant & 0x3f);
10065 %}
10066
10067 ins_pipe(ialu_reg_shift);
10068 %}
10069
10070 // This pattern is automatically generated from aarch64_ad.m4
10071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10072 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10073 immL0 zero, iRegL src1, immI src2) %{
10074 match(Set dst (SubL zero (LShiftL src1 src2)));
10075
10076 ins_cost(1.9 * INSN_COST);
10077 format %{ "neg $dst, $src1, LSL $src2" %}
10078
10079 ins_encode %{
10080 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10081 Assembler::LSL, $src2$$constant & 0x3f);
10082 %}
10083
10084 ins_pipe(ialu_reg_shift);
10085 %}
10086
10087 // This pattern is automatically generated from aarch64_ad.m4
10088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10089 instruct AndI_reg_not_reg(iRegINoSp dst,
10090 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10091 match(Set dst (AndI src1 (XorI src2 m1)));
10092 ins_cost(INSN_COST);
10093 format %{ "bicw $dst, $src1, $src2" %}
10094
10095 ins_encode %{
10096 __ bicw(as_Register($dst$$reg),
10097 as_Register($src1$$reg),
10098 as_Register($src2$$reg),
10099 Assembler::LSL, 0);
10100 %}
10101
10102 ins_pipe(ialu_reg_reg);
10103 %}
10104
10105 // This pattern is automatically generated from aarch64_ad.m4
10106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10107 instruct AndL_reg_not_reg(iRegLNoSp dst,
10108 iRegL src1, iRegL src2, immL_M1 m1) %{
10109 match(Set dst (AndL src1 (XorL src2 m1)));
10110 ins_cost(INSN_COST);
10111 format %{ "bic $dst, $src1, $src2" %}
10112
10113 ins_encode %{
10114 __ bic(as_Register($dst$$reg),
10115 as_Register($src1$$reg),
10116 as_Register($src2$$reg),
10117 Assembler::LSL, 0);
10118 %}
10119
10120 ins_pipe(ialu_reg_reg);
10121 %}
10122
10123 // This pattern is automatically generated from aarch64_ad.m4
10124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10125 instruct OrI_reg_not_reg(iRegINoSp dst,
10126 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10127 match(Set dst (OrI src1 (XorI src2 m1)));
10128 ins_cost(INSN_COST);
10129 format %{ "ornw $dst, $src1, $src2" %}
10130
10131 ins_encode %{
10132 __ ornw(as_Register($dst$$reg),
10133 as_Register($src1$$reg),
10134 as_Register($src2$$reg),
10135 Assembler::LSL, 0);
10136 %}
10137
10138 ins_pipe(ialu_reg_reg);
10139 %}
10140
10141 // This pattern is automatically generated from aarch64_ad.m4
10142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10143 instruct OrL_reg_not_reg(iRegLNoSp dst,
10144 iRegL src1, iRegL src2, immL_M1 m1) %{
10145 match(Set dst (OrL src1 (XorL src2 m1)));
10146 ins_cost(INSN_COST);
10147 format %{ "orn $dst, $src1, $src2" %}
10148
10149 ins_encode %{
10150 __ orn(as_Register($dst$$reg),
10151 as_Register($src1$$reg),
10152 as_Register($src2$$reg),
10153 Assembler::LSL, 0);
10154 %}
10155
10156 ins_pipe(ialu_reg_reg);
10157 %}
10158
10159 // This pattern is automatically generated from aarch64_ad.m4
10160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10161 instruct XorI_reg_not_reg(iRegINoSp dst,
10162 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10163 match(Set dst (XorI m1 (XorI src2 src1)));
10164 ins_cost(INSN_COST);
10165 format %{ "eonw $dst, $src1, $src2" %}
10166
10167 ins_encode %{
10168 __ eonw(as_Register($dst$$reg),
10169 as_Register($src1$$reg),
10170 as_Register($src2$$reg),
10171 Assembler::LSL, 0);
10172 %}
10173
10174 ins_pipe(ialu_reg_reg);
10175 %}
10176
10177 // This pattern is automatically generated from aarch64_ad.m4
10178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10179 instruct XorL_reg_not_reg(iRegLNoSp dst,
10180 iRegL src1, iRegL src2, immL_M1 m1) %{
10181 match(Set dst (XorL m1 (XorL src2 src1)));
10182 ins_cost(INSN_COST);
10183 format %{ "eon $dst, $src1, $src2" %}
10184
10185 ins_encode %{
10186 __ eon(as_Register($dst$$reg),
10187 as_Register($src1$$reg),
10188 as_Register($src2$$reg),
10189 Assembler::LSL, 0);
10190 %}
10191
10192 ins_pipe(ialu_reg_reg);
10193 %}
10194
10195 // This pattern is automatically generated from aarch64_ad.m4
10196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10197 // val & (-1 ^ (val >>> shift)) ==> bicw
10198 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10199 iRegIorL2I src1, iRegIorL2I src2,
10200 immI src3, immI_M1 src4) %{
10201 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10202 ins_cost(1.9 * INSN_COST);
10203 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10204
10205 ins_encode %{
10206 __ bicw(as_Register($dst$$reg),
10207 as_Register($src1$$reg),
10208 as_Register($src2$$reg),
10209 Assembler::LSR,
10210 $src3$$constant & 0x1f);
10211 %}
10212
10213 ins_pipe(ialu_reg_reg_shift);
10214 %}
10215
10216 // This pattern is automatically generated from aarch64_ad.m4
10217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10218 // val & (-1 ^ (val >>> shift)) ==> bic
10219 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10220 iRegL src1, iRegL src2,
10221 immI src3, immL_M1 src4) %{
10222 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10223 ins_cost(1.9 * INSN_COST);
10224 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10225
10226 ins_encode %{
10227 __ bic(as_Register($dst$$reg),
10228 as_Register($src1$$reg),
10229 as_Register($src2$$reg),
10230 Assembler::LSR,
10231 $src3$$constant & 0x3f);
10232 %}
10233
10234 ins_pipe(ialu_reg_reg_shift);
10235 %}
10236
10237 // This pattern is automatically generated from aarch64_ad.m4
10238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10239 // val & (-1 ^ (val >> shift)) ==> bicw
10240 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10241 iRegIorL2I src1, iRegIorL2I src2,
10242 immI src3, immI_M1 src4) %{
10243 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10244 ins_cost(1.9 * INSN_COST);
10245 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10246
10247 ins_encode %{
10248 __ bicw(as_Register($dst$$reg),
10249 as_Register($src1$$reg),
10250 as_Register($src2$$reg),
10251 Assembler::ASR,
10252 $src3$$constant & 0x1f);
10253 %}
10254
10255 ins_pipe(ialu_reg_reg_shift);
10256 %}
10257
10258 // This pattern is automatically generated from aarch64_ad.m4
10259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10260 // val & (-1 ^ (val >> shift)) ==> bic
10261 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10262 iRegL src1, iRegL src2,
10263 immI src3, immL_M1 src4) %{
10264 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10265 ins_cost(1.9 * INSN_COST);
10266 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10267
10268 ins_encode %{
10269 __ bic(as_Register($dst$$reg),
10270 as_Register($src1$$reg),
10271 as_Register($src2$$reg),
10272 Assembler::ASR,
10273 $src3$$constant & 0x3f);
10274 %}
10275
10276 ins_pipe(ialu_reg_reg_shift);
10277 %}
10278
10279 // This pattern is automatically generated from aarch64_ad.m4
10280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10281 // val & (-1 ^ (val ror shift)) ==> bicw
10282 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10283 iRegIorL2I src1, iRegIorL2I src2,
10284 immI src3, immI_M1 src4) %{
10285 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10286 ins_cost(1.9 * INSN_COST);
10287 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10288
10289 ins_encode %{
10290 __ bicw(as_Register($dst$$reg),
10291 as_Register($src1$$reg),
10292 as_Register($src2$$reg),
10293 Assembler::ROR,
10294 $src3$$constant & 0x1f);
10295 %}
10296
10297 ins_pipe(ialu_reg_reg_shift);
10298 %}
10299
10300 // This pattern is automatically generated from aarch64_ad.m4
10301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10302 // val & (-1 ^ (val ror shift)) ==> bic
10303 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10304 iRegL src1, iRegL src2,
10305 immI src3, immL_M1 src4) %{
10306 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10307 ins_cost(1.9 * INSN_COST);
10308 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10309
10310 ins_encode %{
10311 __ bic(as_Register($dst$$reg),
10312 as_Register($src1$$reg),
10313 as_Register($src2$$reg),
10314 Assembler::ROR,
10315 $src3$$constant & 0x3f);
10316 %}
10317
10318 ins_pipe(ialu_reg_reg_shift);
10319 %}
10320
10321 // This pattern is automatically generated from aarch64_ad.m4
10322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10323 // val & (-1 ^ (val << shift)) ==> bicw
10324 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10325 iRegIorL2I src1, iRegIorL2I src2,
10326 immI src3, immI_M1 src4) %{
10327 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10328 ins_cost(1.9 * INSN_COST);
10329 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10330
10331 ins_encode %{
10332 __ bicw(as_Register($dst$$reg),
10333 as_Register($src1$$reg),
10334 as_Register($src2$$reg),
10335 Assembler::LSL,
10336 $src3$$constant & 0x1f);
10337 %}
10338
10339 ins_pipe(ialu_reg_reg_shift);
10340 %}
10341
10342 // This pattern is automatically generated from aarch64_ad.m4
10343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10344 // val & (-1 ^ (val << shift)) ==> bic
10345 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10346 iRegL src1, iRegL src2,
10347 immI src3, immL_M1 src4) %{
10348 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10349 ins_cost(1.9 * INSN_COST);
10350 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10351
10352 ins_encode %{
10353 __ bic(as_Register($dst$$reg),
10354 as_Register($src1$$reg),
10355 as_Register($src2$$reg),
10356 Assembler::LSL,
10357 $src3$$constant & 0x3f);
10358 %}
10359
10360 ins_pipe(ialu_reg_reg_shift);
10361 %}
10362
10363 // This pattern is automatically generated from aarch64_ad.m4
10364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10365 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10366 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10367 iRegIorL2I src1, iRegIorL2I src2,
10368 immI src3, immI_M1 src4) %{
10369 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10370 ins_cost(1.9 * INSN_COST);
10371 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10372
10373 ins_encode %{
10374 __ eonw(as_Register($dst$$reg),
10375 as_Register($src1$$reg),
10376 as_Register($src2$$reg),
10377 Assembler::LSR,
10378 $src3$$constant & 0x1f);
10379 %}
10380
10381 ins_pipe(ialu_reg_reg_shift);
10382 %}
10383
10384 // This pattern is automatically generated from aarch64_ad.m4
10385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10386 // val ^ (-1 ^ (val >>> shift)) ==> eon
10387 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10388 iRegL src1, iRegL src2,
10389 immI src3, immL_M1 src4) %{
10390 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10391 ins_cost(1.9 * INSN_COST);
10392 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10393
10394 ins_encode %{
10395 __ eon(as_Register($dst$$reg),
10396 as_Register($src1$$reg),
10397 as_Register($src2$$reg),
10398 Assembler::LSR,
10399 $src3$$constant & 0x3f);
10400 %}
10401
10402 ins_pipe(ialu_reg_reg_shift);
10403 %}
10404
10405 // This pattern is automatically generated from aarch64_ad.m4
10406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10407 // val ^ (-1 ^ (val >> shift)) ==> eonw
10408 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10409 iRegIorL2I src1, iRegIorL2I src2,
10410 immI src3, immI_M1 src4) %{
10411 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10412 ins_cost(1.9 * INSN_COST);
10413 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10414
10415 ins_encode %{
10416 __ eonw(as_Register($dst$$reg),
10417 as_Register($src1$$reg),
10418 as_Register($src2$$reg),
10419 Assembler::ASR,
10420 $src3$$constant & 0x1f);
10421 %}
10422
10423 ins_pipe(ialu_reg_reg_shift);
10424 %}
10425
10426 // This pattern is automatically generated from aarch64_ad.m4
10427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10428 // val ^ (-1 ^ (val >> shift)) ==> eon
10429 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10430 iRegL src1, iRegL src2,
10431 immI src3, immL_M1 src4) %{
10432 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10433 ins_cost(1.9 * INSN_COST);
10434 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10435
10436 ins_encode %{
10437 __ eon(as_Register($dst$$reg),
10438 as_Register($src1$$reg),
10439 as_Register($src2$$reg),
10440 Assembler::ASR,
10441 $src3$$constant & 0x3f);
10442 %}
10443
10444 ins_pipe(ialu_reg_reg_shift);
10445 %}
10446
10447 // This pattern is automatically generated from aarch64_ad.m4
10448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10449 // val ^ (-1 ^ (val ror shift)) ==> eonw
10450 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10451 iRegIorL2I src1, iRegIorL2I src2,
10452 immI src3, immI_M1 src4) %{
10453 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10454 ins_cost(1.9 * INSN_COST);
10455 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10456
10457 ins_encode %{
10458 __ eonw(as_Register($dst$$reg),
10459 as_Register($src1$$reg),
10460 as_Register($src2$$reg),
10461 Assembler::ROR,
10462 $src3$$constant & 0x1f);
10463 %}
10464
10465 ins_pipe(ialu_reg_reg_shift);
10466 %}
10467
10468 // This pattern is automatically generated from aarch64_ad.m4
10469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10470 // val ^ (-1 ^ (val ror shift)) ==> eon
10471 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10472 iRegL src1, iRegL src2,
10473 immI src3, immL_M1 src4) %{
10474 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10475 ins_cost(1.9 * INSN_COST);
10476 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10477
10478 ins_encode %{
10479 __ eon(as_Register($dst$$reg),
10480 as_Register($src1$$reg),
10481 as_Register($src2$$reg),
10482 Assembler::ROR,
10483 $src3$$constant & 0x3f);
10484 %}
10485
10486 ins_pipe(ialu_reg_reg_shift);
10487 %}
10488
10489 // This pattern is automatically generated from aarch64_ad.m4
10490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10491 // val ^ (-1 ^ (val << shift)) ==> eonw
10492 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10493 iRegIorL2I src1, iRegIorL2I src2,
10494 immI src3, immI_M1 src4) %{
10495 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10496 ins_cost(1.9 * INSN_COST);
10497 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10498
10499 ins_encode %{
10500 __ eonw(as_Register($dst$$reg),
10501 as_Register($src1$$reg),
10502 as_Register($src2$$reg),
10503 Assembler::LSL,
10504 $src3$$constant & 0x1f);
10505 %}
10506
10507 ins_pipe(ialu_reg_reg_shift);
10508 %}
10509
10510 // This pattern is automatically generated from aarch64_ad.m4
10511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10512 // val ^ (-1 ^ (val << shift)) ==> eon
10513 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10514 iRegL src1, iRegL src2,
10515 immI src3, immL_M1 src4) %{
10516 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10517 ins_cost(1.9 * INSN_COST);
10518 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10519
10520 ins_encode %{
10521 __ eon(as_Register($dst$$reg),
10522 as_Register($src1$$reg),
10523 as_Register($src2$$reg),
10524 Assembler::LSL,
10525 $src3$$constant & 0x3f);
10526 %}
10527
10528 ins_pipe(ialu_reg_reg_shift);
10529 %}
10530
10531 // This pattern is automatically generated from aarch64_ad.m4
10532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10533 // val | (-1 ^ (val >>> shift)) ==> ornw
10534 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10535 iRegIorL2I src1, iRegIorL2I src2,
10536 immI src3, immI_M1 src4) %{
10537 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10538 ins_cost(1.9 * INSN_COST);
10539 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10540
10541 ins_encode %{
10542 __ ornw(as_Register($dst$$reg),
10543 as_Register($src1$$reg),
10544 as_Register($src2$$reg),
10545 Assembler::LSR,
10546 $src3$$constant & 0x1f);
10547 %}
10548
10549 ins_pipe(ialu_reg_reg_shift);
10550 %}
10551
10552 // This pattern is automatically generated from aarch64_ad.m4
10553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10554 // val | (-1 ^ (val >>> shift)) ==> orn
10555 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10556 iRegL src1, iRegL src2,
10557 immI src3, immL_M1 src4) %{
10558 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10559 ins_cost(1.9 * INSN_COST);
10560 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10561
10562 ins_encode %{
10563 __ orn(as_Register($dst$$reg),
10564 as_Register($src1$$reg),
10565 as_Register($src2$$reg),
10566 Assembler::LSR,
10567 $src3$$constant & 0x3f);
10568 %}
10569
10570 ins_pipe(ialu_reg_reg_shift);
10571 %}
10572
10573 // This pattern is automatically generated from aarch64_ad.m4
10574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10575 // val | (-1 ^ (val >> shift)) ==> ornw
10576 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10577 iRegIorL2I src1, iRegIorL2I src2,
10578 immI src3, immI_M1 src4) %{
10579 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10580 ins_cost(1.9 * INSN_COST);
10581 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10582
10583 ins_encode %{
10584 __ ornw(as_Register($dst$$reg),
10585 as_Register($src1$$reg),
10586 as_Register($src2$$reg),
10587 Assembler::ASR,
10588 $src3$$constant & 0x1f);
10589 %}
10590
10591 ins_pipe(ialu_reg_reg_shift);
10592 %}
10593
10594 // This pattern is automatically generated from aarch64_ad.m4
10595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10596 // val | (-1 ^ (val >> shift)) ==> orn
10597 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10598 iRegL src1, iRegL src2,
10599 immI src3, immL_M1 src4) %{
10600 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10601 ins_cost(1.9 * INSN_COST);
10602 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10603
10604 ins_encode %{
10605 __ orn(as_Register($dst$$reg),
10606 as_Register($src1$$reg),
10607 as_Register($src2$$reg),
10608 Assembler::ASR,
10609 $src3$$constant & 0x3f);
10610 %}
10611
10612 ins_pipe(ialu_reg_reg_shift);
10613 %}
10614
10615 // This pattern is automatically generated from aarch64_ad.m4
10616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10617 // val | (-1 ^ (val ror shift)) ==> ornw
10618 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10619 iRegIorL2I src1, iRegIorL2I src2,
10620 immI src3, immI_M1 src4) %{
10621 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10622 ins_cost(1.9 * INSN_COST);
10623 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10624
10625 ins_encode %{
10626 __ ornw(as_Register($dst$$reg),
10627 as_Register($src1$$reg),
10628 as_Register($src2$$reg),
10629 Assembler::ROR,
10630 $src3$$constant & 0x1f);
10631 %}
10632
10633 ins_pipe(ialu_reg_reg_shift);
10634 %}
10635
10636 // This pattern is automatically generated from aarch64_ad.m4
10637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10638 // val | (-1 ^ (val ror shift)) ==> orn
10639 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10640 iRegL src1, iRegL src2,
10641 immI src3, immL_M1 src4) %{
10642 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10643 ins_cost(1.9 * INSN_COST);
10644 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10645
10646 ins_encode %{
10647 __ orn(as_Register($dst$$reg),
10648 as_Register($src1$$reg),
10649 as_Register($src2$$reg),
10650 Assembler::ROR,
10651 $src3$$constant & 0x3f);
10652 %}
10653
10654 ins_pipe(ialu_reg_reg_shift);
10655 %}
10656
10657 // This pattern is automatically generated from aarch64_ad.m4
10658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10659 // val | (-1 ^ (val << shift)) ==> ornw
10660 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10661 iRegIorL2I src1, iRegIorL2I src2,
10662 immI src3, immI_M1 src4) %{
10663 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10664 ins_cost(1.9 * INSN_COST);
10665 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10666
10667 ins_encode %{
10668 __ ornw(as_Register($dst$$reg),
10669 as_Register($src1$$reg),
10670 as_Register($src2$$reg),
10671 Assembler::LSL,
10672 $src3$$constant & 0x1f);
10673 %}
10674
10675 ins_pipe(ialu_reg_reg_shift);
10676 %}
10677
10678 // This pattern is automatically generated from aarch64_ad.m4
10679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10680 // val | (-1 ^ (val << shift)) ==> orn
10681 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10682 iRegL src1, iRegL src2,
10683 immI src3, immL_M1 src4) %{
10684 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10685 ins_cost(1.9 * INSN_COST);
10686 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10687
10688 ins_encode %{
10689 __ orn(as_Register($dst$$reg),
10690 as_Register($src1$$reg),
10691 as_Register($src2$$reg),
10692 Assembler::LSL,
10693 $src3$$constant & 0x3f);
10694 %}
10695
10696 ins_pipe(ialu_reg_reg_shift);
10697 %}
10698
10699 // This pattern is automatically generated from aarch64_ad.m4
10700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10701 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10702 iRegIorL2I src1, iRegIorL2I src2,
10703 immI src3) %{
10704 match(Set dst (AndI src1 (URShiftI src2 src3)));
10705
10706 ins_cost(1.9 * INSN_COST);
10707 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10708
10709 ins_encode %{
10710 __ andw(as_Register($dst$$reg),
10711 as_Register($src1$$reg),
10712 as_Register($src2$$reg),
10713 Assembler::LSR,
10714 $src3$$constant & 0x1f);
10715 %}
10716
10717 ins_pipe(ialu_reg_reg_shift);
10718 %}
10719
10720 // This pattern is automatically generated from aarch64_ad.m4
10721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10722 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10723 iRegL src1, iRegL src2,
10724 immI src3) %{
10725 match(Set dst (AndL src1 (URShiftL src2 src3)));
10726
10727 ins_cost(1.9 * INSN_COST);
10728 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10729
10730 ins_encode %{
10731 __ andr(as_Register($dst$$reg),
10732 as_Register($src1$$reg),
10733 as_Register($src2$$reg),
10734 Assembler::LSR,
10735 $src3$$constant & 0x3f);
10736 %}
10737
10738 ins_pipe(ialu_reg_reg_shift);
10739 %}
10740
10741 // This pattern is automatically generated from aarch64_ad.m4
10742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10743 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10744 iRegIorL2I src1, iRegIorL2I src2,
10745 immI src3) %{
10746 match(Set dst (AndI src1 (RShiftI src2 src3)));
10747
10748 ins_cost(1.9 * INSN_COST);
10749 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10750
10751 ins_encode %{
10752 __ andw(as_Register($dst$$reg),
10753 as_Register($src1$$reg),
10754 as_Register($src2$$reg),
10755 Assembler::ASR,
10756 $src3$$constant & 0x1f);
10757 %}
10758
10759 ins_pipe(ialu_reg_reg_shift);
10760 %}
10761
10762 // This pattern is automatically generated from aarch64_ad.m4
10763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10764 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10765 iRegL src1, iRegL src2,
10766 immI src3) %{
10767 match(Set dst (AndL src1 (RShiftL src2 src3)));
10768
10769 ins_cost(1.9 * INSN_COST);
10770 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10771
10772 ins_encode %{
10773 __ andr(as_Register($dst$$reg),
10774 as_Register($src1$$reg),
10775 as_Register($src2$$reg),
10776 Assembler::ASR,
10777 $src3$$constant & 0x3f);
10778 %}
10779
10780 ins_pipe(ialu_reg_reg_shift);
10781 %}
10782
10783 // This pattern is automatically generated from aarch64_ad.m4
10784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10785 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10786 iRegIorL2I src1, iRegIorL2I src2,
10787 immI src3) %{
10788 match(Set dst (AndI src1 (LShiftI src2 src3)));
10789
10790 ins_cost(1.9 * INSN_COST);
10791 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10792
10793 ins_encode %{
10794 __ andw(as_Register($dst$$reg),
10795 as_Register($src1$$reg),
10796 as_Register($src2$$reg),
10797 Assembler::LSL,
10798 $src3$$constant & 0x1f);
10799 %}
10800
10801 ins_pipe(ialu_reg_reg_shift);
10802 %}
10803
10804 // This pattern is automatically generated from aarch64_ad.m4
10805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10806 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10807 iRegL src1, iRegL src2,
10808 immI src3) %{
10809 match(Set dst (AndL src1 (LShiftL src2 src3)));
10810
10811 ins_cost(1.9 * INSN_COST);
10812 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10813
10814 ins_encode %{
10815 __ andr(as_Register($dst$$reg),
10816 as_Register($src1$$reg),
10817 as_Register($src2$$reg),
10818 Assembler::LSL,
10819 $src3$$constant & 0x3f);
10820 %}
10821
10822 ins_pipe(ialu_reg_reg_shift);
10823 %}
10824
10825 // This pattern is automatically generated from aarch64_ad.m4
10826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10827 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10828 iRegIorL2I src1, iRegIorL2I src2,
10829 immI src3) %{
10830 match(Set dst (AndI src1 (RotateRight src2 src3)));
10831
10832 ins_cost(1.9 * INSN_COST);
10833 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10834
10835 ins_encode %{
10836 __ andw(as_Register($dst$$reg),
10837 as_Register($src1$$reg),
10838 as_Register($src2$$reg),
10839 Assembler::ROR,
10840 $src3$$constant & 0x1f);
10841 %}
10842
10843 ins_pipe(ialu_reg_reg_shift);
10844 %}
10845
10846 // This pattern is automatically generated from aarch64_ad.m4
10847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10848 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10849 iRegL src1, iRegL src2,
10850 immI src3) %{
10851 match(Set dst (AndL src1 (RotateRight src2 src3)));
10852
10853 ins_cost(1.9 * INSN_COST);
10854 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10855
10856 ins_encode %{
10857 __ andr(as_Register($dst$$reg),
10858 as_Register($src1$$reg),
10859 as_Register($src2$$reg),
10860 Assembler::ROR,
10861 $src3$$constant & 0x3f);
10862 %}
10863
10864 ins_pipe(ialu_reg_reg_shift);
10865 %}
10866
10867 // This pattern is automatically generated from aarch64_ad.m4
10868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10869 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10870 iRegIorL2I src1, iRegIorL2I src2,
10871 immI src3) %{
10872 match(Set dst (XorI src1 (URShiftI src2 src3)));
10873
10874 ins_cost(1.9 * INSN_COST);
10875 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10876
10877 ins_encode %{
10878 __ eorw(as_Register($dst$$reg),
10879 as_Register($src1$$reg),
10880 as_Register($src2$$reg),
10881 Assembler::LSR,
10882 $src3$$constant & 0x1f);
10883 %}
10884
10885 ins_pipe(ialu_reg_reg_shift);
10886 %}
10887
10888 // This pattern is automatically generated from aarch64_ad.m4
10889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10890 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10891 iRegL src1, iRegL src2,
10892 immI src3) %{
10893 match(Set dst (XorL src1 (URShiftL src2 src3)));
10894
10895 ins_cost(1.9 * INSN_COST);
10896 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10897
10898 ins_encode %{
10899 __ eor(as_Register($dst$$reg),
10900 as_Register($src1$$reg),
10901 as_Register($src2$$reg),
10902 Assembler::LSR,
10903 $src3$$constant & 0x3f);
10904 %}
10905
10906 ins_pipe(ialu_reg_reg_shift);
10907 %}
10908
10909 // This pattern is automatically generated from aarch64_ad.m4
10910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10911 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10912 iRegIorL2I src1, iRegIorL2I src2,
10913 immI src3) %{
10914 match(Set dst (XorI src1 (RShiftI src2 src3)));
10915
10916 ins_cost(1.9 * INSN_COST);
10917 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10918
10919 ins_encode %{
10920 __ eorw(as_Register($dst$$reg),
10921 as_Register($src1$$reg),
10922 as_Register($src2$$reg),
10923 Assembler::ASR,
10924 $src3$$constant & 0x1f);
10925 %}
10926
10927 ins_pipe(ialu_reg_reg_shift);
10928 %}
10929
10930 // This pattern is automatically generated from aarch64_ad.m4
10931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10932 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10933 iRegL src1, iRegL src2,
10934 immI src3) %{
10935 match(Set dst (XorL src1 (RShiftL src2 src3)));
10936
10937 ins_cost(1.9 * INSN_COST);
10938 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10939
10940 ins_encode %{
10941 __ eor(as_Register($dst$$reg),
10942 as_Register($src1$$reg),
10943 as_Register($src2$$reg),
10944 Assembler::ASR,
10945 $src3$$constant & 0x3f);
10946 %}
10947
10948 ins_pipe(ialu_reg_reg_shift);
10949 %}
10950
10951 // This pattern is automatically generated from aarch64_ad.m4
10952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10953 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10954 iRegIorL2I src1, iRegIorL2I src2,
10955 immI src3) %{
10956 match(Set dst (XorI src1 (LShiftI src2 src3)));
10957
10958 ins_cost(1.9 * INSN_COST);
10959 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10960
10961 ins_encode %{
10962 __ eorw(as_Register($dst$$reg),
10963 as_Register($src1$$reg),
10964 as_Register($src2$$reg),
10965 Assembler::LSL,
10966 $src3$$constant & 0x1f);
10967 %}
10968
10969 ins_pipe(ialu_reg_reg_shift);
10970 %}
10971
10972 // This pattern is automatically generated from aarch64_ad.m4
10973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10974 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10975 iRegL src1, iRegL src2,
10976 immI src3) %{
10977 match(Set dst (XorL src1 (LShiftL src2 src3)));
10978
10979 ins_cost(1.9 * INSN_COST);
10980 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10981
10982 ins_encode %{
10983 __ eor(as_Register($dst$$reg),
10984 as_Register($src1$$reg),
10985 as_Register($src2$$reg),
10986 Assembler::LSL,
10987 $src3$$constant & 0x3f);
10988 %}
10989
10990 ins_pipe(ialu_reg_reg_shift);
10991 %}
10992
10993 // This pattern is automatically generated from aarch64_ad.m4
10994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10995 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10996 iRegIorL2I src1, iRegIorL2I src2,
10997 immI src3) %{
10998 match(Set dst (XorI src1 (RotateRight src2 src3)));
10999
11000 ins_cost(1.9 * INSN_COST);
11001 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11002
11003 ins_encode %{
11004 __ eorw(as_Register($dst$$reg),
11005 as_Register($src1$$reg),
11006 as_Register($src2$$reg),
11007 Assembler::ROR,
11008 $src3$$constant & 0x1f);
11009 %}
11010
11011 ins_pipe(ialu_reg_reg_shift);
11012 %}
11013
11014 // This pattern is automatically generated from aarch64_ad.m4
11015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11016 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11017 iRegL src1, iRegL src2,
11018 immI src3) %{
11019 match(Set dst (XorL src1 (RotateRight src2 src3)));
11020
11021 ins_cost(1.9 * INSN_COST);
11022 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11023
11024 ins_encode %{
11025 __ eor(as_Register($dst$$reg),
11026 as_Register($src1$$reg),
11027 as_Register($src2$$reg),
11028 Assembler::ROR,
11029 $src3$$constant & 0x3f);
11030 %}
11031
11032 ins_pipe(ialu_reg_reg_shift);
11033 %}
11034
11035 // This pattern is automatically generated from aarch64_ad.m4
11036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11037 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11038 iRegIorL2I src1, iRegIorL2I src2,
11039 immI src3) %{
11040 match(Set dst (OrI src1 (URShiftI src2 src3)));
11041
11042 ins_cost(1.9 * INSN_COST);
11043 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11044
11045 ins_encode %{
11046 __ orrw(as_Register($dst$$reg),
11047 as_Register($src1$$reg),
11048 as_Register($src2$$reg),
11049 Assembler::LSR,
11050 $src3$$constant & 0x1f);
11051 %}
11052
11053 ins_pipe(ialu_reg_reg_shift);
11054 %}
11055
11056 // This pattern is automatically generated from aarch64_ad.m4
11057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11058 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11059 iRegL src1, iRegL src2,
11060 immI src3) %{
11061 match(Set dst (OrL src1 (URShiftL src2 src3)));
11062
11063 ins_cost(1.9 * INSN_COST);
11064 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11065
11066 ins_encode %{
11067 __ orr(as_Register($dst$$reg),
11068 as_Register($src1$$reg),
11069 as_Register($src2$$reg),
11070 Assembler::LSR,
11071 $src3$$constant & 0x3f);
11072 %}
11073
11074 ins_pipe(ialu_reg_reg_shift);
11075 %}
11076
11077 // This pattern is automatically generated from aarch64_ad.m4
11078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11079 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11080 iRegIorL2I src1, iRegIorL2I src2,
11081 immI src3) %{
11082 match(Set dst (OrI src1 (RShiftI src2 src3)));
11083
11084 ins_cost(1.9 * INSN_COST);
11085 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11086
11087 ins_encode %{
11088 __ orrw(as_Register($dst$$reg),
11089 as_Register($src1$$reg),
11090 as_Register($src2$$reg),
11091 Assembler::ASR,
11092 $src3$$constant & 0x1f);
11093 %}
11094
11095 ins_pipe(ialu_reg_reg_shift);
11096 %}
11097
11098 // This pattern is automatically generated from aarch64_ad.m4
11099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11100 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11101 iRegL src1, iRegL src2,
11102 immI src3) %{
11103 match(Set dst (OrL src1 (RShiftL src2 src3)));
11104
11105 ins_cost(1.9 * INSN_COST);
11106 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11107
11108 ins_encode %{
11109 __ orr(as_Register($dst$$reg),
11110 as_Register($src1$$reg),
11111 as_Register($src2$$reg),
11112 Assembler::ASR,
11113 $src3$$constant & 0x3f);
11114 %}
11115
11116 ins_pipe(ialu_reg_reg_shift);
11117 %}
11118
11119 // This pattern is automatically generated from aarch64_ad.m4
11120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11121 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11122 iRegIorL2I src1, iRegIorL2I src2,
11123 immI src3) %{
11124 match(Set dst (OrI src1 (LShiftI src2 src3)));
11125
11126 ins_cost(1.9 * INSN_COST);
11127 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11128
11129 ins_encode %{
11130 __ orrw(as_Register($dst$$reg),
11131 as_Register($src1$$reg),
11132 as_Register($src2$$reg),
11133 Assembler::LSL,
11134 $src3$$constant & 0x1f);
11135 %}
11136
11137 ins_pipe(ialu_reg_reg_shift);
11138 %}
11139
11140 // This pattern is automatically generated from aarch64_ad.m4
11141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11142 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11143 iRegL src1, iRegL src2,
11144 immI src3) %{
11145 match(Set dst (OrL src1 (LShiftL src2 src3)));
11146
11147 ins_cost(1.9 * INSN_COST);
11148 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11149
11150 ins_encode %{
11151 __ orr(as_Register($dst$$reg),
11152 as_Register($src1$$reg),
11153 as_Register($src2$$reg),
11154 Assembler::LSL,
11155 $src3$$constant & 0x3f);
11156 %}
11157
11158 ins_pipe(ialu_reg_reg_shift);
11159 %}
11160
11161 // This pattern is automatically generated from aarch64_ad.m4
11162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11163 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11164 iRegIorL2I src1, iRegIorL2I src2,
11165 immI src3) %{
11166 match(Set dst (OrI src1 (RotateRight src2 src3)));
11167
11168 ins_cost(1.9 * INSN_COST);
11169 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11170
11171 ins_encode %{
11172 __ orrw(as_Register($dst$$reg),
11173 as_Register($src1$$reg),
11174 as_Register($src2$$reg),
11175 Assembler::ROR,
11176 $src3$$constant & 0x1f);
11177 %}
11178
11179 ins_pipe(ialu_reg_reg_shift);
11180 %}
11181
11182 // This pattern is automatically generated from aarch64_ad.m4
11183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11184 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11185 iRegL src1, iRegL src2,
11186 immI src3) %{
11187 match(Set dst (OrL src1 (RotateRight src2 src3)));
11188
11189 ins_cost(1.9 * INSN_COST);
11190 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11191
11192 ins_encode %{
11193 __ orr(as_Register($dst$$reg),
11194 as_Register($src1$$reg),
11195 as_Register($src2$$reg),
11196 Assembler::ROR,
11197 $src3$$constant & 0x3f);
11198 %}
11199
11200 ins_pipe(ialu_reg_reg_shift);
11201 %}
11202
11203 // This pattern is automatically generated from aarch64_ad.m4
11204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11205 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11206 iRegIorL2I src1, iRegIorL2I src2,
11207 immI src3) %{
11208 match(Set dst (AddI src1 (URShiftI src2 src3)));
11209
11210 ins_cost(1.9 * INSN_COST);
11211 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11212
11213 ins_encode %{
11214 __ addw(as_Register($dst$$reg),
11215 as_Register($src1$$reg),
11216 as_Register($src2$$reg),
11217 Assembler::LSR,
11218 $src3$$constant & 0x1f);
11219 %}
11220
11221 ins_pipe(ialu_reg_reg_shift);
11222 %}
11223
11224 // This pattern is automatically generated from aarch64_ad.m4
11225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11226 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11227 iRegL src1, iRegL src2,
11228 immI src3) %{
11229 match(Set dst (AddL src1 (URShiftL src2 src3)));
11230
11231 ins_cost(1.9 * INSN_COST);
11232 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11233
11234 ins_encode %{
11235 __ add(as_Register($dst$$reg),
11236 as_Register($src1$$reg),
11237 as_Register($src2$$reg),
11238 Assembler::LSR,
11239 $src3$$constant & 0x3f);
11240 %}
11241
11242 ins_pipe(ialu_reg_reg_shift);
11243 %}
11244
11245 // This pattern is automatically generated from aarch64_ad.m4
11246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11247 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11248 iRegIorL2I src1, iRegIorL2I src2,
11249 immI src3) %{
11250 match(Set dst (AddI src1 (RShiftI src2 src3)));
11251
11252 ins_cost(1.9 * INSN_COST);
11253 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11254
11255 ins_encode %{
11256 __ addw(as_Register($dst$$reg),
11257 as_Register($src1$$reg),
11258 as_Register($src2$$reg),
11259 Assembler::ASR,
11260 $src3$$constant & 0x1f);
11261 %}
11262
11263 ins_pipe(ialu_reg_reg_shift);
11264 %}
11265
11266 // This pattern is automatically generated from aarch64_ad.m4
11267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11268 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11269 iRegL src1, iRegL src2,
11270 immI src3) %{
11271 match(Set dst (AddL src1 (RShiftL src2 src3)));
11272
11273 ins_cost(1.9 * INSN_COST);
11274 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11275
11276 ins_encode %{
11277 __ add(as_Register($dst$$reg),
11278 as_Register($src1$$reg),
11279 as_Register($src2$$reg),
11280 Assembler::ASR,
11281 $src3$$constant & 0x3f);
11282 %}
11283
11284 ins_pipe(ialu_reg_reg_shift);
11285 %}
11286
11287 // This pattern is automatically generated from aarch64_ad.m4
11288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11289 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11290 iRegIorL2I src1, iRegIorL2I src2,
11291 immI src3) %{
11292 match(Set dst (AddI src1 (LShiftI src2 src3)));
11293
11294 ins_cost(1.9 * INSN_COST);
11295 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11296
11297 ins_encode %{
11298 __ addw(as_Register($dst$$reg),
11299 as_Register($src1$$reg),
11300 as_Register($src2$$reg),
11301 Assembler::LSL,
11302 $src3$$constant & 0x1f);
11303 %}
11304
11305 ins_pipe(ialu_reg_reg_shift);
11306 %}
11307
11308 // This pattern is automatically generated from aarch64_ad.m4
11309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11310 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11311 iRegL src1, iRegL src2,
11312 immI src3) %{
11313 match(Set dst (AddL src1 (LShiftL src2 src3)));
11314
11315 ins_cost(1.9 * INSN_COST);
11316 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11317
11318 ins_encode %{
11319 __ add(as_Register($dst$$reg),
11320 as_Register($src1$$reg),
11321 as_Register($src2$$reg),
11322 Assembler::LSL,
11323 $src3$$constant & 0x3f);
11324 %}
11325
11326 ins_pipe(ialu_reg_reg_shift);
11327 %}
11328
11329 // This pattern is automatically generated from aarch64_ad.m4
11330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11331 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11332 iRegIorL2I src1, iRegIorL2I src2,
11333 immI src3) %{
11334 match(Set dst (SubI src1 (URShiftI src2 src3)));
11335
11336 ins_cost(1.9 * INSN_COST);
11337 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11338
11339 ins_encode %{
11340 __ subw(as_Register($dst$$reg),
11341 as_Register($src1$$reg),
11342 as_Register($src2$$reg),
11343 Assembler::LSR,
11344 $src3$$constant & 0x1f);
11345 %}
11346
11347 ins_pipe(ialu_reg_reg_shift);
11348 %}
11349
11350 // This pattern is automatically generated from aarch64_ad.m4
11351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11352 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11353 iRegL src1, iRegL src2,
11354 immI src3) %{
11355 match(Set dst (SubL src1 (URShiftL src2 src3)));
11356
11357 ins_cost(1.9 * INSN_COST);
11358 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11359
11360 ins_encode %{
11361 __ sub(as_Register($dst$$reg),
11362 as_Register($src1$$reg),
11363 as_Register($src2$$reg),
11364 Assembler::LSR,
11365 $src3$$constant & 0x3f);
11366 %}
11367
11368 ins_pipe(ialu_reg_reg_shift);
11369 %}
11370
11371 // This pattern is automatically generated from aarch64_ad.m4
11372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11373 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11374 iRegIorL2I src1, iRegIorL2I src2,
11375 immI src3) %{
11376 match(Set dst (SubI src1 (RShiftI src2 src3)));
11377
11378 ins_cost(1.9 * INSN_COST);
11379 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11380
11381 ins_encode %{
11382 __ subw(as_Register($dst$$reg),
11383 as_Register($src1$$reg),
11384 as_Register($src2$$reg),
11385 Assembler::ASR,
11386 $src3$$constant & 0x1f);
11387 %}
11388
11389 ins_pipe(ialu_reg_reg_shift);
11390 %}
11391
11392 // This pattern is automatically generated from aarch64_ad.m4
11393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11394 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11395 iRegL src1, iRegL src2,
11396 immI src3) %{
11397 match(Set dst (SubL src1 (RShiftL src2 src3)));
11398
11399 ins_cost(1.9 * INSN_COST);
11400 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11401
11402 ins_encode %{
11403 __ sub(as_Register($dst$$reg),
11404 as_Register($src1$$reg),
11405 as_Register($src2$$reg),
11406 Assembler::ASR,
11407 $src3$$constant & 0x3f);
11408 %}
11409
11410 ins_pipe(ialu_reg_reg_shift);
11411 %}
11412
11413 // This pattern is automatically generated from aarch64_ad.m4
11414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11415 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11416 iRegIorL2I src1, iRegIorL2I src2,
11417 immI src3) %{
11418 match(Set dst (SubI src1 (LShiftI src2 src3)));
11419
11420 ins_cost(1.9 * INSN_COST);
11421 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11422
11423 ins_encode %{
11424 __ subw(as_Register($dst$$reg),
11425 as_Register($src1$$reg),
11426 as_Register($src2$$reg),
11427 Assembler::LSL,
11428 $src3$$constant & 0x1f);
11429 %}
11430
11431 ins_pipe(ialu_reg_reg_shift);
11432 %}
11433
11434 // This pattern is automatically generated from aarch64_ad.m4
11435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11436 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11437 iRegL src1, iRegL src2,
11438 immI src3) %{
11439 match(Set dst (SubL src1 (LShiftL src2 src3)));
11440
11441 ins_cost(1.9 * INSN_COST);
11442 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11443
11444 ins_encode %{
11445 __ sub(as_Register($dst$$reg),
11446 as_Register($src1$$reg),
11447 as_Register($src2$$reg),
11448 Assembler::LSL,
11449 $src3$$constant & 0x3f);
11450 %}
11451
11452 ins_pipe(ialu_reg_reg_shift);
11453 %}
11454
11455 // This pattern is automatically generated from aarch64_ad.m4
11456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11457
11458 // Shift Left followed by Shift Right.
11459 // This idiom is used by the compiler for the i2b bytecode etc.
11460 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11461 %{
11462 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11463 ins_cost(INSN_COST * 2);
11464 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11465 ins_encode %{
11466 int lshift = $lshift_count$$constant & 63;
11467 int rshift = $rshift_count$$constant & 63;
11468 int s = 63 - lshift;
11469 int r = (rshift - lshift) & 63;
11470 __ sbfm(as_Register($dst$$reg),
11471 as_Register($src$$reg),
11472 r, s);
11473 %}
11474
11475 ins_pipe(ialu_reg_shift);
11476 %}
11477
11478 // This pattern is automatically generated from aarch64_ad.m4
11479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11480
11481 // Shift Left followed by Shift Right.
11482 // This idiom is used by the compiler for the i2b bytecode etc.
11483 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11484 %{
11485 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11486 ins_cost(INSN_COST * 2);
11487 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11488 ins_encode %{
11489 int lshift = $lshift_count$$constant & 31;
11490 int rshift = $rshift_count$$constant & 31;
11491 int s = 31 - lshift;
11492 int r = (rshift - lshift) & 31;
11493 __ sbfmw(as_Register($dst$$reg),
11494 as_Register($src$$reg),
11495 r, s);
11496 %}
11497
11498 ins_pipe(ialu_reg_shift);
11499 %}
11500
11501 // This pattern is automatically generated from aarch64_ad.m4
11502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11503
11504 // Shift Left followed by Shift Right.
11505 // This idiom is used by the compiler for the i2b bytecode etc.
11506 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11507 %{
11508 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11509 ins_cost(INSN_COST * 2);
11510 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11511 ins_encode %{
11512 int lshift = $lshift_count$$constant & 63;
11513 int rshift = $rshift_count$$constant & 63;
11514 int s = 63 - lshift;
11515 int r = (rshift - lshift) & 63;
11516 __ ubfm(as_Register($dst$$reg),
11517 as_Register($src$$reg),
11518 r, s);
11519 %}
11520
11521 ins_pipe(ialu_reg_shift);
11522 %}
11523
11524 // This pattern is automatically generated from aarch64_ad.m4
11525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11526
11527 // Shift Left followed by Shift Right.
11528 // This idiom is used by the compiler for the i2b bytecode etc.
11529 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11530 %{
11531 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11532 ins_cost(INSN_COST * 2);
11533 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11534 ins_encode %{
11535 int lshift = $lshift_count$$constant & 31;
11536 int rshift = $rshift_count$$constant & 31;
11537 int s = 31 - lshift;
11538 int r = (rshift - lshift) & 31;
11539 __ ubfmw(as_Register($dst$$reg),
11540 as_Register($src$$reg),
11541 r, s);
11542 %}
11543
11544 ins_pipe(ialu_reg_shift);
11545 %}
11546
11547 // Bitfield extract with shift & mask
11548
11549 // This pattern is automatically generated from aarch64_ad.m4
11550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11551 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11552 %{
11553 match(Set dst (AndI (URShiftI src rshift) mask));
11554 // Make sure we are not going to exceed what ubfxw can do.
11555 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11556
11557 ins_cost(INSN_COST);
11558 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11559 ins_encode %{
11560 int rshift = $rshift$$constant & 31;
11561 intptr_t mask = $mask$$constant;
11562 int width = exact_log2(mask+1);
11563 __ ubfxw(as_Register($dst$$reg),
11564 as_Register($src$$reg), rshift, width);
11565 %}
11566 ins_pipe(ialu_reg_shift);
11567 %}
11568
11569 // This pattern is automatically generated from aarch64_ad.m4
11570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11571 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11572 %{
11573 match(Set dst (AndL (URShiftL src rshift) mask));
11574 // Make sure we are not going to exceed what ubfx can do.
11575 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11576
11577 ins_cost(INSN_COST);
11578 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11579 ins_encode %{
11580 int rshift = $rshift$$constant & 63;
11581 intptr_t mask = $mask$$constant;
11582 int width = exact_log2_long(mask+1);
11583 __ ubfx(as_Register($dst$$reg),
11584 as_Register($src$$reg), rshift, width);
11585 %}
11586 ins_pipe(ialu_reg_shift);
11587 %}
11588
11589
11590 // This pattern is automatically generated from aarch64_ad.m4
11591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11592
11593 // We can use ubfx when extending an And with a mask when we know mask
11594 // is positive. We know that because immI_bitmask guarantees it.
11595 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11596 %{
11597 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11598 // Make sure we are not going to exceed what ubfxw can do.
11599 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11600
11601 ins_cost(INSN_COST * 2);
11602 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11603 ins_encode %{
11604 int rshift = $rshift$$constant & 31;
11605 intptr_t mask = $mask$$constant;
11606 int width = exact_log2(mask+1);
11607 __ ubfx(as_Register($dst$$reg),
11608 as_Register($src$$reg), rshift, width);
11609 %}
11610 ins_pipe(ialu_reg_shift);
11611 %}
11612
11613
11614 // This pattern is automatically generated from aarch64_ad.m4
11615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11616
11617 // We can use ubfiz when masking by a positive number and then left shifting the result.
11618 // We know that the mask is positive because immI_bitmask guarantees it.
11619 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11620 %{
11621 match(Set dst (LShiftI (AndI src mask) lshift));
11622 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11623
11624 ins_cost(INSN_COST);
11625 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11626 ins_encode %{
11627 int lshift = $lshift$$constant & 31;
11628 intptr_t mask = $mask$$constant;
11629 int width = exact_log2(mask+1);
11630 __ ubfizw(as_Register($dst$$reg),
11631 as_Register($src$$reg), lshift, width);
11632 %}
11633 ins_pipe(ialu_reg_shift);
11634 %}
11635
11636 // This pattern is automatically generated from aarch64_ad.m4
11637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11638
11639 // We can use ubfiz when masking by a positive number and then left shifting the result.
11640 // We know that the mask is positive because immL_bitmask guarantees it.
11641 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11642 %{
11643 match(Set dst (LShiftL (AndL src mask) lshift));
11644 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11645
11646 ins_cost(INSN_COST);
11647 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11648 ins_encode %{
11649 int lshift = $lshift$$constant & 63;
11650 intptr_t mask = $mask$$constant;
11651 int width = exact_log2_long(mask+1);
11652 __ ubfiz(as_Register($dst$$reg),
11653 as_Register($src$$reg), lshift, width);
11654 %}
11655 ins_pipe(ialu_reg_shift);
11656 %}
11657
11658 // This pattern is automatically generated from aarch64_ad.m4
11659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11660
11661 // We can use ubfiz when masking by a positive number and then left shifting the result.
11662 // We know that the mask is positive because immI_bitmask guarantees it.
11663 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11664 %{
11665 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11666 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11667
11668 ins_cost(INSN_COST);
11669 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11670 ins_encode %{
11671 int lshift = $lshift$$constant & 31;
11672 intptr_t mask = $mask$$constant;
11673 int width = exact_log2(mask+1);
11674 __ ubfizw(as_Register($dst$$reg),
11675 as_Register($src$$reg), lshift, width);
11676 %}
11677 ins_pipe(ialu_reg_shift);
11678 %}
11679
11680 // This pattern is automatically generated from aarch64_ad.m4
11681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11682
11683 // We can use ubfiz when masking by a positive number and then left shifting the result.
11684 // We know that the mask is positive because immL_bitmask guarantees it.
11685 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11686 %{
11687 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11688 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11689
11690 ins_cost(INSN_COST);
11691 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11692 ins_encode %{
11693 int lshift = $lshift$$constant & 63;
11694 intptr_t mask = $mask$$constant;
11695 int width = exact_log2_long(mask+1);
11696 __ ubfiz(as_Register($dst$$reg),
11697 as_Register($src$$reg), lshift, width);
11698 %}
11699 ins_pipe(ialu_reg_shift);
11700 %}
11701
11702
11703 // This pattern is automatically generated from aarch64_ad.m4
11704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11705
11706 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11707 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11708 %{
11709 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11710 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11711
11712 ins_cost(INSN_COST);
11713 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11714 ins_encode %{
11715 int lshift = $lshift$$constant & 63;
11716 intptr_t mask = $mask$$constant;
11717 int width = exact_log2(mask+1);
11718 __ ubfiz(as_Register($dst$$reg),
11719 as_Register($src$$reg), lshift, width);
11720 %}
11721 ins_pipe(ialu_reg_shift);
11722 %}
11723
11724 // This pattern is automatically generated from aarch64_ad.m4
11725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11726
11727 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11728 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11729 %{
11730 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11731 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11732
11733 ins_cost(INSN_COST);
11734 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11735 ins_encode %{
11736 int lshift = $lshift$$constant & 31;
11737 intptr_t mask = $mask$$constant;
11738 int width = exact_log2(mask+1);
11739 __ ubfiz(as_Register($dst$$reg),
11740 as_Register($src$$reg), lshift, width);
11741 %}
11742 ins_pipe(ialu_reg_shift);
11743 %}
11744
11745 // This pattern is automatically generated from aarch64_ad.m4
11746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11747
11748 // Can skip int2long conversions after AND with small bitmask
11749 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11750 %{
11751 match(Set dst (ConvI2L (AndI src msk)));
11752 ins_cost(INSN_COST);
11753 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11754 ins_encode %{
11755 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11756 %}
11757 ins_pipe(ialu_reg_shift);
11758 %}
11759
11760
11761 // Rotations
11762
11763 // This pattern is automatically generated from aarch64_ad.m4
11764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11765 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11766 %{
11767 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11768 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11769
11770 ins_cost(INSN_COST);
11771 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11772
11773 ins_encode %{
11774 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11775 $rshift$$constant & 63);
11776 %}
11777 ins_pipe(ialu_reg_reg_extr);
11778 %}
11779
11780
11781 // This pattern is automatically generated from aarch64_ad.m4
11782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11783 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11784 %{
11785 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11786 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11787
11788 ins_cost(INSN_COST);
11789 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11790
11791 ins_encode %{
11792 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11793 $rshift$$constant & 31);
11794 %}
11795 ins_pipe(ialu_reg_reg_extr);
11796 %}
11797
11798
11799 // This pattern is automatically generated from aarch64_ad.m4
11800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11801 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11802 %{
11803 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11804 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11805
11806 ins_cost(INSN_COST);
11807 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11808
11809 ins_encode %{
11810 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11811 $rshift$$constant & 63);
11812 %}
11813 ins_pipe(ialu_reg_reg_extr);
11814 %}
11815
11816
11817 // This pattern is automatically generated from aarch64_ad.m4
11818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11819 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11820 %{
11821 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11822 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11823
11824 ins_cost(INSN_COST);
11825 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11826
11827 ins_encode %{
11828 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11829 $rshift$$constant & 31);
11830 %}
11831 ins_pipe(ialu_reg_reg_extr);
11832 %}
11833
11834 // This pattern is automatically generated from aarch64_ad.m4
11835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11836 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11837 %{
11838 match(Set dst (RotateRight src shift));
11839
11840 ins_cost(INSN_COST);
11841 format %{ "ror $dst, $src, $shift" %}
11842
11843 ins_encode %{
11844 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11845 $shift$$constant & 0x1f);
11846 %}
11847 ins_pipe(ialu_reg_reg_vshift);
11848 %}
11849
11850 // This pattern is automatically generated from aarch64_ad.m4
11851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11852 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11853 %{
11854 match(Set dst (RotateRight src shift));
11855
11856 ins_cost(INSN_COST);
11857 format %{ "ror $dst, $src, $shift" %}
11858
11859 ins_encode %{
11860 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11861 $shift$$constant & 0x3f);
11862 %}
11863 ins_pipe(ialu_reg_reg_vshift);
11864 %}
11865
11866 // This pattern is automatically generated from aarch64_ad.m4
11867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11868 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11869 %{
11870 match(Set dst (RotateRight src shift));
11871
11872 ins_cost(INSN_COST);
11873 format %{ "ror $dst, $src, $shift" %}
11874
11875 ins_encode %{
11876 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11877 %}
11878 ins_pipe(ialu_reg_reg_vshift);
11879 %}
11880
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11884 %{
11885 match(Set dst (RotateRight src shift));
11886
11887 ins_cost(INSN_COST);
11888 format %{ "ror $dst, $src, $shift" %}
11889
11890 ins_encode %{
11891 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11892 %}
11893 ins_pipe(ialu_reg_reg_vshift);
11894 %}
11895
11896 // This pattern is automatically generated from aarch64_ad.m4
11897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11898 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11899 %{
11900 match(Set dst (RotateLeft src shift));
11901
11902 ins_cost(INSN_COST);
11903 format %{ "rol $dst, $src, $shift" %}
11904
11905 ins_encode %{
11906 __ subw(rscratch1, zr, as_Register($shift$$reg));
11907 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11908 %}
11909 ins_pipe(ialu_reg_reg_vshift);
11910 %}
11911
11912 // This pattern is automatically generated from aarch64_ad.m4
11913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11914 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11915 %{
11916 match(Set dst (RotateLeft src shift));
11917
11918 ins_cost(INSN_COST);
11919 format %{ "rol $dst, $src, $shift" %}
11920
11921 ins_encode %{
11922 __ subw(rscratch1, zr, as_Register($shift$$reg));
11923 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11924 %}
11925 ins_pipe(ialu_reg_reg_vshift);
11926 %}
11927
11928
11929 // Add/subtract (extended)
11930
11931 // This pattern is automatically generated from aarch64_ad.m4
11932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11933 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11934 %{
11935 match(Set dst (AddL src1 (ConvI2L src2)));
11936 ins_cost(INSN_COST);
11937 format %{ "add $dst, $src1, $src2, sxtw" %}
11938
11939 ins_encode %{
11940 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11941 as_Register($src2$$reg), ext::sxtw);
11942 %}
11943 ins_pipe(ialu_reg_reg);
11944 %}
11945
11946 // This pattern is automatically generated from aarch64_ad.m4
11947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11948 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11949 %{
11950 match(Set dst (SubL src1 (ConvI2L src2)));
11951 ins_cost(INSN_COST);
11952 format %{ "sub $dst, $src1, $src2, sxtw" %}
11953
11954 ins_encode %{
11955 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11956 as_Register($src2$$reg), ext::sxtw);
11957 %}
11958 ins_pipe(ialu_reg_reg);
11959 %}
11960
11961 // This pattern is automatically generated from aarch64_ad.m4
11962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11963 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11964 %{
11965 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11966 ins_cost(INSN_COST);
11967 format %{ "add $dst, $src1, $src2, sxth" %}
11968
11969 ins_encode %{
11970 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11971 as_Register($src2$$reg), ext::sxth);
11972 %}
11973 ins_pipe(ialu_reg_reg);
11974 %}
11975
11976 // This pattern is automatically generated from aarch64_ad.m4
11977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11978 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11979 %{
11980 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11981 ins_cost(INSN_COST);
11982 format %{ "add $dst, $src1, $src2, sxtb" %}
11983
11984 ins_encode %{
11985 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11986 as_Register($src2$$reg), ext::sxtb);
11987 %}
11988 ins_pipe(ialu_reg_reg);
11989 %}
11990
11991 // This pattern is automatically generated from aarch64_ad.m4
11992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11993 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11994 %{
11995 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11996 ins_cost(INSN_COST);
11997 format %{ "add $dst, $src1, $src2, uxtb" %}
11998
11999 ins_encode %{
12000 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12001 as_Register($src2$$reg), ext::uxtb);
12002 %}
12003 ins_pipe(ialu_reg_reg);
12004 %}
12005
12006 // This pattern is automatically generated from aarch64_ad.m4
12007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12008 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12009 %{
12010 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12011 ins_cost(INSN_COST);
12012 format %{ "add $dst, $src1, $src2, sxth" %}
12013
12014 ins_encode %{
12015 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12016 as_Register($src2$$reg), ext::sxth);
12017 %}
12018 ins_pipe(ialu_reg_reg);
12019 %}
12020
12021 // This pattern is automatically generated from aarch64_ad.m4
12022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12023 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12024 %{
12025 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12026 ins_cost(INSN_COST);
12027 format %{ "add $dst, $src1, $src2, sxtw" %}
12028
12029 ins_encode %{
12030 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12031 as_Register($src2$$reg), ext::sxtw);
12032 %}
12033 ins_pipe(ialu_reg_reg);
12034 %}
12035
12036 // This pattern is automatically generated from aarch64_ad.m4
12037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12038 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12039 %{
12040 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12041 ins_cost(INSN_COST);
12042 format %{ "add $dst, $src1, $src2, sxtb" %}
12043
12044 ins_encode %{
12045 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12046 as_Register($src2$$reg), ext::sxtb);
12047 %}
12048 ins_pipe(ialu_reg_reg);
12049 %}
12050
12051 // This pattern is automatically generated from aarch64_ad.m4
12052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12053 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12054 %{
12055 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12056 ins_cost(INSN_COST);
12057 format %{ "add $dst, $src1, $src2, uxtb" %}
12058
12059 ins_encode %{
12060 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12061 as_Register($src2$$reg), ext::uxtb);
12062 %}
12063 ins_pipe(ialu_reg_reg);
12064 %}
12065
12066 // This pattern is automatically generated from aarch64_ad.m4
12067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12068 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12069 %{
12070 match(Set dst (AddI src1 (AndI src2 mask)));
12071 ins_cost(INSN_COST);
12072 format %{ "addw $dst, $src1, $src2, uxtb" %}
12073
12074 ins_encode %{
12075 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12076 as_Register($src2$$reg), ext::uxtb);
12077 %}
12078 ins_pipe(ialu_reg_reg);
12079 %}
12080
12081 // This pattern is automatically generated from aarch64_ad.m4
12082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12083 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12084 %{
12085 match(Set dst (AddI src1 (AndI src2 mask)));
12086 ins_cost(INSN_COST);
12087 format %{ "addw $dst, $src1, $src2, uxth" %}
12088
12089 ins_encode %{
12090 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12091 as_Register($src2$$reg), ext::uxth);
12092 %}
12093 ins_pipe(ialu_reg_reg);
12094 %}
12095
12096 // This pattern is automatically generated from aarch64_ad.m4
12097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12098 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12099 %{
12100 match(Set dst (AddL src1 (AndL src2 mask)));
12101 ins_cost(INSN_COST);
12102 format %{ "add $dst, $src1, $src2, uxtb" %}
12103
12104 ins_encode %{
12105 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12106 as_Register($src2$$reg), ext::uxtb);
12107 %}
12108 ins_pipe(ialu_reg_reg);
12109 %}
12110
12111 // This pattern is automatically generated from aarch64_ad.m4
12112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12113 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12114 %{
12115 match(Set dst (AddL src1 (AndL src2 mask)));
12116 ins_cost(INSN_COST);
12117 format %{ "add $dst, $src1, $src2, uxth" %}
12118
12119 ins_encode %{
12120 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12121 as_Register($src2$$reg), ext::uxth);
12122 %}
12123 ins_pipe(ialu_reg_reg);
12124 %}
12125
12126 // This pattern is automatically generated from aarch64_ad.m4
12127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12128 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12129 %{
12130 match(Set dst (AddL src1 (AndL src2 mask)));
12131 ins_cost(INSN_COST);
12132 format %{ "add $dst, $src1, $src2, uxtw" %}
12133
12134 ins_encode %{
12135 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12136 as_Register($src2$$reg), ext::uxtw);
12137 %}
12138 ins_pipe(ialu_reg_reg);
12139 %}
12140
12141 // This pattern is automatically generated from aarch64_ad.m4
12142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12143 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12144 %{
12145 match(Set dst (SubI src1 (AndI src2 mask)));
12146 ins_cost(INSN_COST);
12147 format %{ "subw $dst, $src1, $src2, uxtb" %}
12148
12149 ins_encode %{
12150 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12151 as_Register($src2$$reg), ext::uxtb);
12152 %}
12153 ins_pipe(ialu_reg_reg);
12154 %}
12155
12156 // This pattern is automatically generated from aarch64_ad.m4
12157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12158 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12159 %{
12160 match(Set dst (SubI src1 (AndI src2 mask)));
12161 ins_cost(INSN_COST);
12162 format %{ "subw $dst, $src1, $src2, uxth" %}
12163
12164 ins_encode %{
12165 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12166 as_Register($src2$$reg), ext::uxth);
12167 %}
12168 ins_pipe(ialu_reg_reg);
12169 %}
12170
12171 // This pattern is automatically generated from aarch64_ad.m4
12172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12173 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12174 %{
12175 match(Set dst (SubL src1 (AndL src2 mask)));
12176 ins_cost(INSN_COST);
12177 format %{ "sub $dst, $src1, $src2, uxtb" %}
12178
12179 ins_encode %{
12180 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12181 as_Register($src2$$reg), ext::uxtb);
12182 %}
12183 ins_pipe(ialu_reg_reg);
12184 %}
12185
12186 // This pattern is automatically generated from aarch64_ad.m4
12187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12188 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12189 %{
12190 match(Set dst (SubL src1 (AndL src2 mask)));
12191 ins_cost(INSN_COST);
12192 format %{ "sub $dst, $src1, $src2, uxth" %}
12193
12194 ins_encode %{
12195 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12196 as_Register($src2$$reg), ext::uxth);
12197 %}
12198 ins_pipe(ialu_reg_reg);
12199 %}
12200
12201 // This pattern is automatically generated from aarch64_ad.m4
12202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12203 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12204 %{
12205 match(Set dst (SubL src1 (AndL src2 mask)));
12206 ins_cost(INSN_COST);
12207 format %{ "sub $dst, $src1, $src2, uxtw" %}
12208
12209 ins_encode %{
12210 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12211 as_Register($src2$$reg), ext::uxtw);
12212 %}
12213 ins_pipe(ialu_reg_reg);
12214 %}
12215
12216
12217 // This pattern is automatically generated from aarch64_ad.m4
12218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12219 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12220 %{
12221 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12222 ins_cost(1.9 * INSN_COST);
12223 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12224
12225 ins_encode %{
12226 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12227 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12228 %}
12229 ins_pipe(ialu_reg_reg_shift);
12230 %}
12231
12232 // This pattern is automatically generated from aarch64_ad.m4
12233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12234 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12235 %{
12236 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12237 ins_cost(1.9 * INSN_COST);
12238 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12239
12240 ins_encode %{
12241 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12242 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12243 %}
12244 ins_pipe(ialu_reg_reg_shift);
12245 %}
12246
12247 // This pattern is automatically generated from aarch64_ad.m4
12248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12249 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12250 %{
12251 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12252 ins_cost(1.9 * INSN_COST);
12253 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12254
12255 ins_encode %{
12256 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12257 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12258 %}
12259 ins_pipe(ialu_reg_reg_shift);
12260 %}
12261
12262 // This pattern is automatically generated from aarch64_ad.m4
12263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12264 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12265 %{
12266 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12267 ins_cost(1.9 * INSN_COST);
12268 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12269
12270 ins_encode %{
12271 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12272 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12273 %}
12274 ins_pipe(ialu_reg_reg_shift);
12275 %}
12276
12277 // This pattern is automatically generated from aarch64_ad.m4
12278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12279 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12280 %{
12281 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12282 ins_cost(1.9 * INSN_COST);
12283 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12284
12285 ins_encode %{
12286 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12287 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12288 %}
12289 ins_pipe(ialu_reg_reg_shift);
12290 %}
12291
12292 // This pattern is automatically generated from aarch64_ad.m4
12293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12294 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12295 %{
12296 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12297 ins_cost(1.9 * INSN_COST);
12298 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12299
12300 ins_encode %{
12301 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12302 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12303 %}
12304 ins_pipe(ialu_reg_reg_shift);
12305 %}
12306
12307 // This pattern is automatically generated from aarch64_ad.m4
12308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12309 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12310 %{
12311 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12312 ins_cost(1.9 * INSN_COST);
12313 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12314
12315 ins_encode %{
12316 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12317 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12318 %}
12319 ins_pipe(ialu_reg_reg_shift);
12320 %}
12321
12322 // This pattern is automatically generated from aarch64_ad.m4
12323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12324 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12325 %{
12326 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12327 ins_cost(1.9 * INSN_COST);
12328 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12329
12330 ins_encode %{
12331 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12332 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12333 %}
12334 ins_pipe(ialu_reg_reg_shift);
12335 %}
12336
12337 // This pattern is automatically generated from aarch64_ad.m4
12338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12339 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12340 %{
12341 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12342 ins_cost(1.9 * INSN_COST);
12343 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12344
12345 ins_encode %{
12346 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12347 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12348 %}
12349 ins_pipe(ialu_reg_reg_shift);
12350 %}
12351
12352 // This pattern is automatically generated from aarch64_ad.m4
12353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12354 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12355 %{
12356 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12357 ins_cost(1.9 * INSN_COST);
12358 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12359
12360 ins_encode %{
12361 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12362 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12363 %}
12364 ins_pipe(ialu_reg_reg_shift);
12365 %}
12366
12367 // This pattern is automatically generated from aarch64_ad.m4
12368 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12369 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12370 %{
12371 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12372 ins_cost(1.9 * INSN_COST);
12373 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12374
12375 ins_encode %{
12376 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12377 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12378 %}
12379 ins_pipe(ialu_reg_reg_shift);
12380 %}
12381
12382 // This pattern is automatically generated from aarch64_ad.m4
12383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12384 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12385 %{
12386 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12387 ins_cost(1.9 * INSN_COST);
12388 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12389
12390 ins_encode %{
12391 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12392 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12393 %}
12394 ins_pipe(ialu_reg_reg_shift);
12395 %}
12396
12397 // This pattern is automatically generated from aarch64_ad.m4
12398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12399 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12400 %{
12401 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12402 ins_cost(1.9 * INSN_COST);
12403 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12404
12405 ins_encode %{
12406 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12407 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12408 %}
12409 ins_pipe(ialu_reg_reg_shift);
12410 %}
12411
12412 // This pattern is automatically generated from aarch64_ad.m4
12413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12414 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12415 %{
12416 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12417 ins_cost(1.9 * INSN_COST);
12418 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12419
12420 ins_encode %{
12421 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12422 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12423 %}
12424 ins_pipe(ialu_reg_reg_shift);
12425 %}
12426
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12430 %{
12431 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12432 ins_cost(1.9 * INSN_COST);
12433 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12434
12435 ins_encode %{
12436 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12437 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12438 %}
12439 ins_pipe(ialu_reg_reg_shift);
12440 %}
12441
12442 // This pattern is automatically generated from aarch64_ad.m4
12443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12444 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12445 %{
12446 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12447 ins_cost(1.9 * INSN_COST);
12448 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12449
12450 ins_encode %{
12451 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12452 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12453 %}
12454 ins_pipe(ialu_reg_reg_shift);
12455 %}
12456
12457 // This pattern is automatically generated from aarch64_ad.m4
12458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12459 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12460 %{
12461 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12462 ins_cost(1.9 * INSN_COST);
12463 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12464
12465 ins_encode %{
12466 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12467 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12468 %}
12469 ins_pipe(ialu_reg_reg_shift);
12470 %}
12471
12472 // This pattern is automatically generated from aarch64_ad.m4
12473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12474 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12475 %{
12476 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12477 ins_cost(1.9 * INSN_COST);
12478 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12479
12480 ins_encode %{
12481 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12482 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12483 %}
12484 ins_pipe(ialu_reg_reg_shift);
12485 %}
12486
12487 // This pattern is automatically generated from aarch64_ad.m4
12488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12489 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12490 %{
12491 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12492 ins_cost(1.9 * INSN_COST);
12493 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12494
12495 ins_encode %{
12496 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12497 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12498 %}
12499 ins_pipe(ialu_reg_reg_shift);
12500 %}
12501
12502 // This pattern is automatically generated from aarch64_ad.m4
12503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12504 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12505 %{
12506 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12507 ins_cost(1.9 * INSN_COST);
12508 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12509
12510 ins_encode %{
12511 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12512 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12513 %}
12514 ins_pipe(ialu_reg_reg_shift);
12515 %}
12516
12517 // This pattern is automatically generated from aarch64_ad.m4
12518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12519 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12520 %{
12521 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12522 ins_cost(1.9 * INSN_COST);
12523 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12524
12525 ins_encode %{
12526 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12527 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12528 %}
12529 ins_pipe(ialu_reg_reg_shift);
12530 %}
12531
12532 // This pattern is automatically generated from aarch64_ad.m4
12533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12534 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12535 %{
12536 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12537 ins_cost(1.9 * INSN_COST);
12538 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12539
12540 ins_encode %{
12541 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12542 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12543 %}
12544 ins_pipe(ialu_reg_reg_shift);
12545 %}
12546
12547 // This pattern is automatically generated from aarch64_ad.m4
12548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12549 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12550 %{
12551 effect(DEF dst, USE src1, USE src2, USE cr);
12552 ins_cost(INSN_COST * 2);
12553 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12554
12555 ins_encode %{
12556 __ cselw($dst$$Register,
12557 $src1$$Register,
12558 $src2$$Register,
12559 Assembler::LT);
12560 %}
12561 ins_pipe(icond_reg_reg);
12562 %}
12563
12564 // This pattern is automatically generated from aarch64_ad.m4
12565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12566 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12567 %{
12568 effect(DEF dst, USE src1, USE src2, USE cr);
12569 ins_cost(INSN_COST * 2);
12570 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12571
12572 ins_encode %{
12573 __ cselw($dst$$Register,
12574 $src1$$Register,
12575 $src2$$Register,
12576 Assembler::GT);
12577 %}
12578 ins_pipe(icond_reg_reg);
12579 %}
12580
12581 // This pattern is automatically generated from aarch64_ad.m4
12582 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12583 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12584 %{
12585 effect(DEF dst, USE src1, USE cr);
12586 ins_cost(INSN_COST * 2);
12587 format %{ "cselw $dst, $src1, zr lt\t" %}
12588
12589 ins_encode %{
12590 __ cselw($dst$$Register,
12591 $src1$$Register,
12592 zr,
12593 Assembler::LT);
12594 %}
12595 ins_pipe(icond_reg);
12596 %}
12597
12598 // This pattern is automatically generated from aarch64_ad.m4
12599 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12600 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12601 %{
12602 effect(DEF dst, USE src1, USE cr);
12603 ins_cost(INSN_COST * 2);
12604 format %{ "cselw $dst, $src1, zr gt\t" %}
12605
12606 ins_encode %{
12607 __ cselw($dst$$Register,
12608 $src1$$Register,
12609 zr,
12610 Assembler::GT);
12611 %}
12612 ins_pipe(icond_reg);
12613 %}
12614
12615 // This pattern is automatically generated from aarch64_ad.m4
12616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12617 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12618 %{
12619 effect(DEF dst, USE src1, USE cr);
12620 ins_cost(INSN_COST * 2);
12621 format %{ "csincw $dst, $src1, zr le\t" %}
12622
12623 ins_encode %{
12624 __ csincw($dst$$Register,
12625 $src1$$Register,
12626 zr,
12627 Assembler::LE);
12628 %}
12629 ins_pipe(icond_reg);
12630 %}
12631
12632 // This pattern is automatically generated from aarch64_ad.m4
12633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12634 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12635 %{
12636 effect(DEF dst, USE src1, USE cr);
12637 ins_cost(INSN_COST * 2);
12638 format %{ "csincw $dst, $src1, zr gt\t" %}
12639
12640 ins_encode %{
12641 __ csincw($dst$$Register,
12642 $src1$$Register,
12643 zr,
12644 Assembler::GT);
12645 %}
12646 ins_pipe(icond_reg);
12647 %}
12648
12649 // This pattern is automatically generated from aarch64_ad.m4
12650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12651 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12652 %{
12653 effect(DEF dst, USE src1, USE cr);
12654 ins_cost(INSN_COST * 2);
12655 format %{ "csinvw $dst, $src1, zr lt\t" %}
12656
12657 ins_encode %{
12658 __ csinvw($dst$$Register,
12659 $src1$$Register,
12660 zr,
12661 Assembler::LT);
12662 %}
12663 ins_pipe(icond_reg);
12664 %}
12665
12666 // This pattern is automatically generated from aarch64_ad.m4
12667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12668 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12669 %{
12670 effect(DEF dst, USE src1, USE cr);
12671 ins_cost(INSN_COST * 2);
12672 format %{ "csinvw $dst, $src1, zr ge\t" %}
12673
12674 ins_encode %{
12675 __ csinvw($dst$$Register,
12676 $src1$$Register,
12677 zr,
12678 Assembler::GE);
12679 %}
12680 ins_pipe(icond_reg);
12681 %}
12682
12683 // This pattern is automatically generated from aarch64_ad.m4
12684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12685 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12686 %{
12687 match(Set dst (MinI src imm));
12688 ins_cost(INSN_COST * 3);
12689 expand %{
12690 rFlagsReg cr;
12691 compI_reg_imm0(cr, src);
12692 cmovI_reg_imm0_lt(dst, src, cr);
12693 %}
12694 %}
12695
12696 // This pattern is automatically generated from aarch64_ad.m4
12697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12698 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12699 %{
12700 match(Set dst (MinI imm src));
12701 ins_cost(INSN_COST * 3);
12702 expand %{
12703 rFlagsReg cr;
12704 compI_reg_imm0(cr, src);
12705 cmovI_reg_imm0_lt(dst, src, cr);
12706 %}
12707 %}
12708
12709 // This pattern is automatically generated from aarch64_ad.m4
12710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12711 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12712 %{
12713 match(Set dst (MinI src imm));
12714 ins_cost(INSN_COST * 3);
12715 expand %{
12716 rFlagsReg cr;
12717 compI_reg_imm0(cr, src);
12718 cmovI_reg_imm1_le(dst, src, cr);
12719 %}
12720 %}
12721
12722 // This pattern is automatically generated from aarch64_ad.m4
12723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12724 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12725 %{
12726 match(Set dst (MinI imm src));
12727 ins_cost(INSN_COST * 3);
12728 expand %{
12729 rFlagsReg cr;
12730 compI_reg_imm0(cr, src);
12731 cmovI_reg_imm1_le(dst, src, cr);
12732 %}
12733 %}
12734
12735 // This pattern is automatically generated from aarch64_ad.m4
12736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12737 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12738 %{
12739 match(Set dst (MinI src imm));
12740 ins_cost(INSN_COST * 3);
12741 expand %{
12742 rFlagsReg cr;
12743 compI_reg_imm0(cr, src);
12744 cmovI_reg_immM1_lt(dst, src, cr);
12745 %}
12746 %}
12747
12748 // This pattern is automatically generated from aarch64_ad.m4
12749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12750 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12751 %{
12752 match(Set dst (MinI imm src));
12753 ins_cost(INSN_COST * 3);
12754 expand %{
12755 rFlagsReg cr;
12756 compI_reg_imm0(cr, src);
12757 cmovI_reg_immM1_lt(dst, src, cr);
12758 %}
12759 %}
12760
12761 // This pattern is automatically generated from aarch64_ad.m4
12762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12763 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12764 %{
12765 match(Set dst (MaxI src imm));
12766 ins_cost(INSN_COST * 3);
12767 expand %{
12768 rFlagsReg cr;
12769 compI_reg_imm0(cr, src);
12770 cmovI_reg_imm0_gt(dst, src, cr);
12771 %}
12772 %}
12773
12774 // This pattern is automatically generated from aarch64_ad.m4
12775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12776 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12777 %{
12778 match(Set dst (MaxI imm src));
12779 ins_cost(INSN_COST * 3);
12780 expand %{
12781 rFlagsReg cr;
12782 compI_reg_imm0(cr, src);
12783 cmovI_reg_imm0_gt(dst, src, cr);
12784 %}
12785 %}
12786
12787 // This pattern is automatically generated from aarch64_ad.m4
12788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12789 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12790 %{
12791 match(Set dst (MaxI src imm));
12792 ins_cost(INSN_COST * 3);
12793 expand %{
12794 rFlagsReg cr;
12795 compI_reg_imm0(cr, src);
12796 cmovI_reg_imm1_gt(dst, src, cr);
12797 %}
12798 %}
12799
12800 // This pattern is automatically generated from aarch64_ad.m4
12801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12802 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12803 %{
12804 match(Set dst (MaxI imm src));
12805 ins_cost(INSN_COST * 3);
12806 expand %{
12807 rFlagsReg cr;
12808 compI_reg_imm0(cr, src);
12809 cmovI_reg_imm1_gt(dst, src, cr);
12810 %}
12811 %}
12812
12813 // This pattern is automatically generated from aarch64_ad.m4
12814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12815 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12816 %{
12817 match(Set dst (MaxI src imm));
12818 ins_cost(INSN_COST * 3);
12819 expand %{
12820 rFlagsReg cr;
12821 compI_reg_imm0(cr, src);
12822 cmovI_reg_immM1_ge(dst, src, cr);
12823 %}
12824 %}
12825
12826 // This pattern is automatically generated from aarch64_ad.m4
12827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12828 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12829 %{
12830 match(Set dst (MaxI imm src));
12831 ins_cost(INSN_COST * 3);
12832 expand %{
12833 rFlagsReg cr;
12834 compI_reg_imm0(cr, src);
12835 cmovI_reg_immM1_ge(dst, src, cr);
12836 %}
12837 %}
12838
12839 // This pattern is automatically generated from aarch64_ad.m4
12840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12841 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12842 %{
12843 match(Set dst (ReverseI src));
12844 ins_cost(INSN_COST);
12845 format %{ "rbitw $dst, $src" %}
12846 ins_encode %{
12847 __ rbitw($dst$$Register, $src$$Register);
12848 %}
12849 ins_pipe(ialu_reg);
12850 %}
12851
12852 // This pattern is automatically generated from aarch64_ad.m4
12853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12854 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12855 %{
12856 match(Set dst (ReverseL src));
12857 ins_cost(INSN_COST);
12858 format %{ "rbit $dst, $src" %}
12859 ins_encode %{
12860 __ rbit($dst$$Register, $src$$Register);
12861 %}
12862 ins_pipe(ialu_reg);
12863 %}
12864
12865
12866 // END This section of the file is automatically generated. Do not edit --------------
12867
12868
12869 // ============================================================================
12870 // Floating Point Arithmetic Instructions
12871
12872 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12873 match(Set dst (AddHF src1 src2));
12874 format %{ "faddh $dst, $src1, $src2" %}
12875 ins_encode %{
12876 __ faddh($dst$$FloatRegister,
12877 $src1$$FloatRegister,
12878 $src2$$FloatRegister);
12879 %}
12880 ins_pipe(fp_dop_reg_reg_s);
12881 %}
12882
12883 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12884 match(Set dst (AddF src1 src2));
12885
12886 ins_cost(INSN_COST * 5);
12887 format %{ "fadds $dst, $src1, $src2" %}
12888
12889 ins_encode %{
12890 __ fadds(as_FloatRegister($dst$$reg),
12891 as_FloatRegister($src1$$reg),
12892 as_FloatRegister($src2$$reg));
12893 %}
12894
12895 ins_pipe(fp_dop_reg_reg_s);
12896 %}
12897
12898 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12899 match(Set dst (AddD src1 src2));
12900
12901 ins_cost(INSN_COST * 5);
12902 format %{ "faddd $dst, $src1, $src2" %}
12903
12904 ins_encode %{
12905 __ faddd(as_FloatRegister($dst$$reg),
12906 as_FloatRegister($src1$$reg),
12907 as_FloatRegister($src2$$reg));
12908 %}
12909
12910 ins_pipe(fp_dop_reg_reg_d);
12911 %}
12912
12913 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12914 match(Set dst (SubHF src1 src2));
12915 format %{ "fsubh $dst, $src1, $src2" %}
12916 ins_encode %{
12917 __ fsubh($dst$$FloatRegister,
12918 $src1$$FloatRegister,
12919 $src2$$FloatRegister);
12920 %}
12921 ins_pipe(fp_dop_reg_reg_s);
12922 %}
12923
12924 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12925 match(Set dst (SubF src1 src2));
12926
12927 ins_cost(INSN_COST * 5);
12928 format %{ "fsubs $dst, $src1, $src2" %}
12929
12930 ins_encode %{
12931 __ fsubs(as_FloatRegister($dst$$reg),
12932 as_FloatRegister($src1$$reg),
12933 as_FloatRegister($src2$$reg));
12934 %}
12935
12936 ins_pipe(fp_dop_reg_reg_s);
12937 %}
12938
12939 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12940 match(Set dst (SubD src1 src2));
12941
12942 ins_cost(INSN_COST * 5);
12943 format %{ "fsubd $dst, $src1, $src2" %}
12944
12945 ins_encode %{
12946 __ fsubd(as_FloatRegister($dst$$reg),
12947 as_FloatRegister($src1$$reg),
12948 as_FloatRegister($src2$$reg));
12949 %}
12950
12951 ins_pipe(fp_dop_reg_reg_d);
12952 %}
12953
12954 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12955 match(Set dst (MulHF src1 src2));
12956 format %{ "fmulh $dst, $src1, $src2" %}
12957 ins_encode %{
12958 __ fmulh($dst$$FloatRegister,
12959 $src1$$FloatRegister,
12960 $src2$$FloatRegister);
12961 %}
12962 ins_pipe(fp_dop_reg_reg_s);
12963 %}
12964
12965 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12966 match(Set dst (MulF src1 src2));
12967
12968 ins_cost(INSN_COST * 6);
12969 format %{ "fmuls $dst, $src1, $src2" %}
12970
12971 ins_encode %{
12972 __ fmuls(as_FloatRegister($dst$$reg),
12973 as_FloatRegister($src1$$reg),
12974 as_FloatRegister($src2$$reg));
12975 %}
12976
12977 ins_pipe(fp_dop_reg_reg_s);
12978 %}
12979
12980 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12981 match(Set dst (MulD src1 src2));
12982
12983 ins_cost(INSN_COST * 6);
12984 format %{ "fmuld $dst, $src1, $src2" %}
12985
12986 ins_encode %{
12987 __ fmuld(as_FloatRegister($dst$$reg),
12988 as_FloatRegister($src1$$reg),
12989 as_FloatRegister($src2$$reg));
12990 %}
12991
12992 ins_pipe(fp_dop_reg_reg_d);
12993 %}
12994
12995 // src1 * src2 + src3 (half-precision float)
12996 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12997 match(Set dst (FmaHF src3 (Binary src1 src2)));
12998 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12999 ins_encode %{
13000 assert(UseFMA, "Needs FMA instructions support.");
13001 __ fmaddh($dst$$FloatRegister,
13002 $src1$$FloatRegister,
13003 $src2$$FloatRegister,
13004 $src3$$FloatRegister);
13005 %}
13006 ins_pipe(pipe_class_default);
13007 %}
13008
13009 // src1 * src2 + src3
13010 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13011 match(Set dst (FmaF src3 (Binary src1 src2)));
13012
13013 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13014
13015 ins_encode %{
13016 assert(UseFMA, "Needs FMA instructions support.");
13017 __ fmadds(as_FloatRegister($dst$$reg),
13018 as_FloatRegister($src1$$reg),
13019 as_FloatRegister($src2$$reg),
13020 as_FloatRegister($src3$$reg));
13021 %}
13022
13023 ins_pipe(pipe_class_default);
13024 %}
13025
13026 // src1 * src2 + src3
13027 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13028 match(Set dst (FmaD src3 (Binary src1 src2)));
13029
13030 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13031
13032 ins_encode %{
13033 assert(UseFMA, "Needs FMA instructions support.");
13034 __ fmaddd(as_FloatRegister($dst$$reg),
13035 as_FloatRegister($src1$$reg),
13036 as_FloatRegister($src2$$reg),
13037 as_FloatRegister($src3$$reg));
13038 %}
13039
13040 ins_pipe(pipe_class_default);
13041 %}
13042
13043 // src1 * (-src2) + src3
13044 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13045 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13046 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13047
13048 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13049
13050 ins_encode %{
13051 assert(UseFMA, "Needs FMA instructions support.");
13052 __ fmsubs(as_FloatRegister($dst$$reg),
13053 as_FloatRegister($src1$$reg),
13054 as_FloatRegister($src2$$reg),
13055 as_FloatRegister($src3$$reg));
13056 %}
13057
13058 ins_pipe(pipe_class_default);
13059 %}
13060
13061 // src1 * (-src2) + src3
13062 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13063 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13064 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13065
13066 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13067
13068 ins_encode %{
13069 assert(UseFMA, "Needs FMA instructions support.");
13070 __ fmsubd(as_FloatRegister($dst$$reg),
13071 as_FloatRegister($src1$$reg),
13072 as_FloatRegister($src2$$reg),
13073 as_FloatRegister($src3$$reg));
13074 %}
13075
13076 ins_pipe(pipe_class_default);
13077 %}
13078
13079 // src1 * (-src2) - src3
13080 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13081 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13082 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13083
13084 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13085
13086 ins_encode %{
13087 assert(UseFMA, "Needs FMA instructions support.");
13088 __ fnmadds(as_FloatRegister($dst$$reg),
13089 as_FloatRegister($src1$$reg),
13090 as_FloatRegister($src2$$reg),
13091 as_FloatRegister($src3$$reg));
13092 %}
13093
13094 ins_pipe(pipe_class_default);
13095 %}
13096
13097 // src1 * (-src2) - src3
13098 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13099 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13100 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13101
13102 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13103
13104 ins_encode %{
13105 assert(UseFMA, "Needs FMA instructions support.");
13106 __ fnmaddd(as_FloatRegister($dst$$reg),
13107 as_FloatRegister($src1$$reg),
13108 as_FloatRegister($src2$$reg),
13109 as_FloatRegister($src3$$reg));
13110 %}
13111
13112 ins_pipe(pipe_class_default);
13113 %}
13114
13115 // src1 * src2 - src3
13116 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13117 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13118
13119 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13120
13121 ins_encode %{
13122 assert(UseFMA, "Needs FMA instructions support.");
13123 __ fnmsubs(as_FloatRegister($dst$$reg),
13124 as_FloatRegister($src1$$reg),
13125 as_FloatRegister($src2$$reg),
13126 as_FloatRegister($src3$$reg));
13127 %}
13128
13129 ins_pipe(pipe_class_default);
13130 %}
13131
13132 // src1 * src2 - src3
13133 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13134 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13135
13136 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13137
13138 ins_encode %{
13139 assert(UseFMA, "Needs FMA instructions support.");
13140 // n.b. insn name should be fnmsubd
13141 __ fnmsub(as_FloatRegister($dst$$reg),
13142 as_FloatRegister($src1$$reg),
13143 as_FloatRegister($src2$$reg),
13144 as_FloatRegister($src3$$reg));
13145 %}
13146
13147 ins_pipe(pipe_class_default);
13148 %}
13149
13150 // Math.max(HH)H (half-precision float)
13151 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13152 match(Set dst (MaxHF src1 src2));
13153 format %{ "fmaxh $dst, $src1, $src2" %}
13154 ins_encode %{
13155 __ fmaxh($dst$$FloatRegister,
13156 $src1$$FloatRegister,
13157 $src2$$FloatRegister);
13158 %}
13159 ins_pipe(fp_dop_reg_reg_s);
13160 %}
13161
13162 // Math.min(HH)H (half-precision float)
13163 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13164 match(Set dst (MinHF src1 src2));
13165 format %{ "fminh $dst, $src1, $src2" %}
13166 ins_encode %{
13167 __ fminh($dst$$FloatRegister,
13168 $src1$$FloatRegister,
13169 $src2$$FloatRegister);
13170 %}
13171 ins_pipe(fp_dop_reg_reg_s);
13172 %}
13173
13174 // Math.max(FF)F
13175 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13176 match(Set dst (MaxF src1 src2));
13177
13178 format %{ "fmaxs $dst, $src1, $src2" %}
13179 ins_encode %{
13180 __ fmaxs(as_FloatRegister($dst$$reg),
13181 as_FloatRegister($src1$$reg),
13182 as_FloatRegister($src2$$reg));
13183 %}
13184
13185 ins_pipe(fp_dop_reg_reg_s);
13186 %}
13187
13188 // Math.min(FF)F
13189 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13190 match(Set dst (MinF src1 src2));
13191
13192 format %{ "fmins $dst, $src1, $src2" %}
13193 ins_encode %{
13194 __ fmins(as_FloatRegister($dst$$reg),
13195 as_FloatRegister($src1$$reg),
13196 as_FloatRegister($src2$$reg));
13197 %}
13198
13199 ins_pipe(fp_dop_reg_reg_s);
13200 %}
13201
13202 // Math.max(DD)D
13203 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13204 match(Set dst (MaxD src1 src2));
13205
13206 format %{ "fmaxd $dst, $src1, $src2" %}
13207 ins_encode %{
13208 __ fmaxd(as_FloatRegister($dst$$reg),
13209 as_FloatRegister($src1$$reg),
13210 as_FloatRegister($src2$$reg));
13211 %}
13212
13213 ins_pipe(fp_dop_reg_reg_d);
13214 %}
13215
13216 // Math.min(DD)D
13217 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13218 match(Set dst (MinD src1 src2));
13219
13220 format %{ "fmind $dst, $src1, $src2" %}
13221 ins_encode %{
13222 __ fmind(as_FloatRegister($dst$$reg),
13223 as_FloatRegister($src1$$reg),
13224 as_FloatRegister($src2$$reg));
13225 %}
13226
13227 ins_pipe(fp_dop_reg_reg_d);
13228 %}
13229
13230 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13231 match(Set dst (DivHF src1 src2));
13232 format %{ "fdivh $dst, $src1, $src2" %}
13233 ins_encode %{
13234 __ fdivh($dst$$FloatRegister,
13235 $src1$$FloatRegister,
13236 $src2$$FloatRegister);
13237 %}
13238 ins_pipe(fp_div_s);
13239 %}
13240
13241 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13242 match(Set dst (DivF src1 src2));
13243
13244 ins_cost(INSN_COST * 18);
13245 format %{ "fdivs $dst, $src1, $src2" %}
13246
13247 ins_encode %{
13248 __ fdivs(as_FloatRegister($dst$$reg),
13249 as_FloatRegister($src1$$reg),
13250 as_FloatRegister($src2$$reg));
13251 %}
13252
13253 ins_pipe(fp_div_s);
13254 %}
13255
13256 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13257 match(Set dst (DivD src1 src2));
13258
13259 ins_cost(INSN_COST * 32);
13260 format %{ "fdivd $dst, $src1, $src2" %}
13261
13262 ins_encode %{
13263 __ fdivd(as_FloatRegister($dst$$reg),
13264 as_FloatRegister($src1$$reg),
13265 as_FloatRegister($src2$$reg));
13266 %}
13267
13268 ins_pipe(fp_div_d);
13269 %}
13270
13271 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13272 match(Set dst (NegF src));
13273
13274 ins_cost(INSN_COST * 3);
13275 format %{ "fneg $dst, $src" %}
13276
13277 ins_encode %{
13278 __ fnegs(as_FloatRegister($dst$$reg),
13279 as_FloatRegister($src$$reg));
13280 %}
13281
13282 ins_pipe(fp_uop_s);
13283 %}
13284
13285 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13286 match(Set dst (NegD src));
13287
13288 ins_cost(INSN_COST * 3);
13289 format %{ "fnegd $dst, $src" %}
13290
13291 ins_encode %{
13292 __ fnegd(as_FloatRegister($dst$$reg),
13293 as_FloatRegister($src$$reg));
13294 %}
13295
13296 ins_pipe(fp_uop_d);
13297 %}
13298
13299 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13300 %{
13301 match(Set dst (AbsI src));
13302
13303 effect(KILL cr);
13304 ins_cost(INSN_COST * 2);
13305 format %{ "cmpw $src, zr\n\t"
13306 "cnegw $dst, $src, Assembler::LT\t# int abs"
13307 %}
13308
13309 ins_encode %{
13310 __ cmpw(as_Register($src$$reg), zr);
13311 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13312 %}
13313 ins_pipe(pipe_class_default);
13314 %}
13315
13316 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13317 %{
13318 match(Set dst (AbsL src));
13319
13320 effect(KILL cr);
13321 ins_cost(INSN_COST * 2);
13322 format %{ "cmp $src, zr\n\t"
13323 "cneg $dst, $src, Assembler::LT\t# long abs"
13324 %}
13325
13326 ins_encode %{
13327 __ cmp(as_Register($src$$reg), zr);
13328 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13329 %}
13330 ins_pipe(pipe_class_default);
13331 %}
13332
13333 instruct absF_reg(vRegF dst, vRegF src) %{
13334 match(Set dst (AbsF src));
13335
13336 ins_cost(INSN_COST * 3);
13337 format %{ "fabss $dst, $src" %}
13338 ins_encode %{
13339 __ fabss(as_FloatRegister($dst$$reg),
13340 as_FloatRegister($src$$reg));
13341 %}
13342
13343 ins_pipe(fp_uop_s);
13344 %}
13345
13346 instruct absD_reg(vRegD dst, vRegD src) %{
13347 match(Set dst (AbsD src));
13348
13349 ins_cost(INSN_COST * 3);
13350 format %{ "fabsd $dst, $src" %}
13351 ins_encode %{
13352 __ fabsd(as_FloatRegister($dst$$reg),
13353 as_FloatRegister($src$$reg));
13354 %}
13355
13356 ins_pipe(fp_uop_d);
13357 %}
13358
13359 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13360 match(Set dst (AbsF (SubF src1 src2)));
13361
13362 ins_cost(INSN_COST * 3);
13363 format %{ "fabds $dst, $src1, $src2" %}
13364 ins_encode %{
13365 __ fabds(as_FloatRegister($dst$$reg),
13366 as_FloatRegister($src1$$reg),
13367 as_FloatRegister($src2$$reg));
13368 %}
13369
13370 ins_pipe(fp_uop_s);
13371 %}
13372
13373 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13374 match(Set dst (AbsD (SubD src1 src2)));
13375
13376 ins_cost(INSN_COST * 3);
13377 format %{ "fabdd $dst, $src1, $src2" %}
13378 ins_encode %{
13379 __ fabdd(as_FloatRegister($dst$$reg),
13380 as_FloatRegister($src1$$reg),
13381 as_FloatRegister($src2$$reg));
13382 %}
13383
13384 ins_pipe(fp_uop_d);
13385 %}
13386
13387 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13388 match(Set dst (SqrtD src));
13389
13390 ins_cost(INSN_COST * 50);
13391 format %{ "fsqrtd $dst, $src" %}
13392 ins_encode %{
13393 __ fsqrtd(as_FloatRegister($dst$$reg),
13394 as_FloatRegister($src$$reg));
13395 %}
13396
13397 ins_pipe(fp_div_s);
13398 %}
13399
13400 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13401 match(Set dst (SqrtF src));
13402
13403 ins_cost(INSN_COST * 50);
13404 format %{ "fsqrts $dst, $src" %}
13405 ins_encode %{
13406 __ fsqrts(as_FloatRegister($dst$$reg),
13407 as_FloatRegister($src$$reg));
13408 %}
13409
13410 ins_pipe(fp_div_d);
13411 %}
13412
13413 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13414 match(Set dst (SqrtHF src));
13415 format %{ "fsqrth $dst, $src" %}
13416 ins_encode %{
13417 __ fsqrth($dst$$FloatRegister,
13418 $src$$FloatRegister);
13419 %}
13420 ins_pipe(fp_div_s);
13421 %}
13422
13423 // Math.rint, floor, ceil
13424 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13425 match(Set dst (RoundDoubleMode src rmode));
13426 format %{ "frint $dst, $src, $rmode" %}
13427 ins_encode %{
13428 switch ($rmode$$constant) {
13429 case RoundDoubleModeNode::rmode_rint:
13430 __ frintnd(as_FloatRegister($dst$$reg),
13431 as_FloatRegister($src$$reg));
13432 break;
13433 case RoundDoubleModeNode::rmode_floor:
13434 __ frintmd(as_FloatRegister($dst$$reg),
13435 as_FloatRegister($src$$reg));
13436 break;
13437 case RoundDoubleModeNode::rmode_ceil:
13438 __ frintpd(as_FloatRegister($dst$$reg),
13439 as_FloatRegister($src$$reg));
13440 break;
13441 }
13442 %}
13443 ins_pipe(fp_uop_d);
13444 %}
13445
13446 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13447 match(Set dst (CopySignD src1 (Binary src2 zero)));
13448 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13449 format %{ "CopySignD $dst $src1 $src2" %}
13450 ins_encode %{
13451 FloatRegister dst = as_FloatRegister($dst$$reg),
13452 src1 = as_FloatRegister($src1$$reg),
13453 src2 = as_FloatRegister($src2$$reg),
13454 zero = as_FloatRegister($zero$$reg);
13455 __ fnegd(dst, zero);
13456 __ bsl(dst, __ T8B, src2, src1);
13457 %}
13458 ins_pipe(fp_uop_d);
13459 %}
13460
13461 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13462 match(Set dst (CopySignF src1 src2));
13463 effect(TEMP_DEF dst, USE src1, USE src2);
13464 format %{ "CopySignF $dst $src1 $src2" %}
13465 ins_encode %{
13466 FloatRegister dst = as_FloatRegister($dst$$reg),
13467 src1 = as_FloatRegister($src1$$reg),
13468 src2 = as_FloatRegister($src2$$reg);
13469 __ movi(dst, __ T2S, 0x80, 24);
13470 __ bsl(dst, __ T8B, src2, src1);
13471 %}
13472 ins_pipe(fp_uop_d);
13473 %}
13474
13475 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13476 match(Set dst (SignumD src (Binary zero one)));
13477 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13478 format %{ "signumD $dst, $src" %}
13479 ins_encode %{
13480 FloatRegister src = as_FloatRegister($src$$reg),
13481 dst = as_FloatRegister($dst$$reg),
13482 zero = as_FloatRegister($zero$$reg),
13483 one = as_FloatRegister($one$$reg);
13484 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13485 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13486 // Bit selection instruction gets bit from "one" for each enabled bit in
13487 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13488 // NaN the whole "src" will be copied because "dst" is zero. For all other
13489 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13490 // from "src", and all other bits are copied from 1.0.
13491 __ bsl(dst, __ T8B, one, src);
13492 %}
13493 ins_pipe(fp_uop_d);
13494 %}
13495
13496 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13497 match(Set dst (SignumF src (Binary zero one)));
13498 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13499 format %{ "signumF $dst, $src" %}
13500 ins_encode %{
13501 FloatRegister src = as_FloatRegister($src$$reg),
13502 dst = as_FloatRegister($dst$$reg),
13503 zero = as_FloatRegister($zero$$reg),
13504 one = as_FloatRegister($one$$reg);
13505 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13506 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13507 // Bit selection instruction gets bit from "one" for each enabled bit in
13508 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13509 // NaN the whole "src" will be copied because "dst" is zero. For all other
13510 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13511 // from "src", and all other bits are copied from 1.0.
13512 __ bsl(dst, __ T8B, one, src);
13513 %}
13514 ins_pipe(fp_uop_d);
13515 %}
13516
13517 instruct onspinwait() %{
13518 match(OnSpinWait);
13519 ins_cost(INSN_COST);
13520
13521 format %{ "onspinwait" %}
13522
13523 ins_encode %{
13524 __ spin_wait();
13525 %}
13526 ins_pipe(pipe_class_empty);
13527 %}
13528
13529 // ============================================================================
13530 // Logical Instructions
13531
13532 // Integer Logical Instructions
13533
13534 // And Instructions
13535
13536
13537 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13538 match(Set dst (AndI src1 src2));
13539
13540 format %{ "andw $dst, $src1, $src2\t# int" %}
13541
13542 ins_cost(INSN_COST);
13543 ins_encode %{
13544 __ andw(as_Register($dst$$reg),
13545 as_Register($src1$$reg),
13546 as_Register($src2$$reg));
13547 %}
13548
13549 ins_pipe(ialu_reg_reg);
13550 %}
13551
13552 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13553 match(Set dst (AndI src1 src2));
13554
13555 format %{ "andsw $dst, $src1, $src2\t# int" %}
13556
13557 ins_cost(INSN_COST);
13558 ins_encode %{
13559 __ andw(as_Register($dst$$reg),
13560 as_Register($src1$$reg),
13561 (uint64_t)($src2$$constant));
13562 %}
13563
13564 ins_pipe(ialu_reg_imm);
13565 %}
13566
13567 // Or Instructions
13568
13569 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13570 match(Set dst (OrI src1 src2));
13571
13572 format %{ "orrw $dst, $src1, $src2\t# int" %}
13573
13574 ins_cost(INSN_COST);
13575 ins_encode %{
13576 __ orrw(as_Register($dst$$reg),
13577 as_Register($src1$$reg),
13578 as_Register($src2$$reg));
13579 %}
13580
13581 ins_pipe(ialu_reg_reg);
13582 %}
13583
13584 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13585 match(Set dst (OrI src1 src2));
13586
13587 format %{ "orrw $dst, $src1, $src2\t# int" %}
13588
13589 ins_cost(INSN_COST);
13590 ins_encode %{
13591 __ orrw(as_Register($dst$$reg),
13592 as_Register($src1$$reg),
13593 (uint64_t)($src2$$constant));
13594 %}
13595
13596 ins_pipe(ialu_reg_imm);
13597 %}
13598
13599 // Xor Instructions
13600
13601 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13602 match(Set dst (XorI src1 src2));
13603
13604 format %{ "eorw $dst, $src1, $src2\t# int" %}
13605
13606 ins_cost(INSN_COST);
13607 ins_encode %{
13608 __ eorw(as_Register($dst$$reg),
13609 as_Register($src1$$reg),
13610 as_Register($src2$$reg));
13611 %}
13612
13613 ins_pipe(ialu_reg_reg);
13614 %}
13615
13616 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13617 match(Set dst (XorI src1 src2));
13618
13619 format %{ "eorw $dst, $src1, $src2\t# int" %}
13620
13621 ins_cost(INSN_COST);
13622 ins_encode %{
13623 __ eorw(as_Register($dst$$reg),
13624 as_Register($src1$$reg),
13625 (uint64_t)($src2$$constant));
13626 %}
13627
13628 ins_pipe(ialu_reg_imm);
13629 %}
13630
13631 // Long Logical Instructions
13632 // TODO
13633
13634 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13635 match(Set dst (AndL src1 src2));
13636
13637 format %{ "and $dst, $src1, $src2\t# int" %}
13638
13639 ins_cost(INSN_COST);
13640 ins_encode %{
13641 __ andr(as_Register($dst$$reg),
13642 as_Register($src1$$reg),
13643 as_Register($src2$$reg));
13644 %}
13645
13646 ins_pipe(ialu_reg_reg);
13647 %}
13648
13649 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13650 match(Set dst (AndL src1 src2));
13651
13652 format %{ "and $dst, $src1, $src2\t# int" %}
13653
13654 ins_cost(INSN_COST);
13655 ins_encode %{
13656 __ andr(as_Register($dst$$reg),
13657 as_Register($src1$$reg),
13658 (uint64_t)($src2$$constant));
13659 %}
13660
13661 ins_pipe(ialu_reg_imm);
13662 %}
13663
13664 // Or Instructions
13665
13666 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13667 match(Set dst (OrL src1 src2));
13668
13669 format %{ "orr $dst, $src1, $src2\t# int" %}
13670
13671 ins_cost(INSN_COST);
13672 ins_encode %{
13673 __ orr(as_Register($dst$$reg),
13674 as_Register($src1$$reg),
13675 as_Register($src2$$reg));
13676 %}
13677
13678 ins_pipe(ialu_reg_reg);
13679 %}
13680
13681 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13682 match(Set dst (OrL src1 src2));
13683
13684 format %{ "orr $dst, $src1, $src2\t# int" %}
13685
13686 ins_cost(INSN_COST);
13687 ins_encode %{
13688 __ orr(as_Register($dst$$reg),
13689 as_Register($src1$$reg),
13690 (uint64_t)($src2$$constant));
13691 %}
13692
13693 ins_pipe(ialu_reg_imm);
13694 %}
13695
13696 // Xor Instructions
13697
13698 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13699 match(Set dst (XorL src1 src2));
13700
13701 format %{ "eor $dst, $src1, $src2\t# int" %}
13702
13703 ins_cost(INSN_COST);
13704 ins_encode %{
13705 __ eor(as_Register($dst$$reg),
13706 as_Register($src1$$reg),
13707 as_Register($src2$$reg));
13708 %}
13709
13710 ins_pipe(ialu_reg_reg);
13711 %}
13712
13713 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13714 match(Set dst (XorL src1 src2));
13715
13716 ins_cost(INSN_COST);
13717 format %{ "eor $dst, $src1, $src2\t# int" %}
13718
13719 ins_encode %{
13720 __ eor(as_Register($dst$$reg),
13721 as_Register($src1$$reg),
13722 (uint64_t)($src2$$constant));
13723 %}
13724
13725 ins_pipe(ialu_reg_imm);
13726 %}
13727
13728 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13729 %{
13730 match(Set dst (ConvI2L src));
13731
13732 ins_cost(INSN_COST);
13733 format %{ "sxtw $dst, $src\t# i2l" %}
13734 ins_encode %{
13735 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13736 %}
13737 ins_pipe(ialu_reg_shift);
13738 %}
13739
13740 // this pattern occurs in bigmath arithmetic
13741 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13742 %{
13743 match(Set dst (AndL (ConvI2L src) mask));
13744
13745 ins_cost(INSN_COST);
13746 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13747 ins_encode %{
13748 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13749 %}
13750
13751 ins_pipe(ialu_reg_shift);
13752 %}
13753
13754 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13755 match(Set dst (ConvL2I src));
13756
13757 ins_cost(INSN_COST);
13758 format %{ "movw $dst, $src \t// l2i" %}
13759
13760 ins_encode %{
13761 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13762 %}
13763
13764 ins_pipe(ialu_reg);
13765 %}
13766
13767 instruct convD2F_reg(vRegF dst, vRegD src) %{
13768 match(Set dst (ConvD2F src));
13769
13770 ins_cost(INSN_COST * 5);
13771 format %{ "fcvtd $dst, $src \t// d2f" %}
13772
13773 ins_encode %{
13774 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13775 %}
13776
13777 ins_pipe(fp_d2f);
13778 %}
13779
13780 instruct convF2D_reg(vRegD dst, vRegF src) %{
13781 match(Set dst (ConvF2D src));
13782
13783 ins_cost(INSN_COST * 5);
13784 format %{ "fcvts $dst, $src \t// f2d" %}
13785
13786 ins_encode %{
13787 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13788 %}
13789
13790 ins_pipe(fp_f2d);
13791 %}
13792
13793 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13794 match(Set dst (ConvF2I src));
13795
13796 ins_cost(INSN_COST * 5);
13797 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13798
13799 ins_encode %{
13800 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13801 %}
13802
13803 ins_pipe(fp_f2i);
13804 %}
13805
13806 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13807 match(Set dst (ConvF2L src));
13808
13809 ins_cost(INSN_COST * 5);
13810 format %{ "fcvtzs $dst, $src \t// f2l" %}
13811
13812 ins_encode %{
13813 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13814 %}
13815
13816 ins_pipe(fp_f2l);
13817 %}
13818
13819 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13820 match(Set dst (ConvF2HF src));
13821 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13822 "smov $dst, $tmp\t# move result from $tmp to $dst"
13823 %}
13824 effect(TEMP tmp);
13825 ins_encode %{
13826 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13827 %}
13828 ins_pipe(pipe_slow);
13829 %}
13830
13831 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13832 match(Set dst (ConvHF2F src));
13833 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13834 "fcvt $dst, $tmp\t# convert half to single precision"
13835 %}
13836 effect(TEMP tmp);
13837 ins_encode %{
13838 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13839 %}
13840 ins_pipe(pipe_slow);
13841 %}
13842
13843 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13844 match(Set dst (ConvI2F src));
13845
13846 ins_cost(INSN_COST * 5);
13847 format %{ "scvtfws $dst, $src \t// i2f" %}
13848
13849 ins_encode %{
13850 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13851 %}
13852
13853 ins_pipe(fp_i2f);
13854 %}
13855
13856 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13857 match(Set dst (ConvL2F src));
13858
13859 ins_cost(INSN_COST * 5);
13860 format %{ "scvtfs $dst, $src \t// l2f" %}
13861
13862 ins_encode %{
13863 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13864 %}
13865
13866 ins_pipe(fp_l2f);
13867 %}
13868
13869 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13870 match(Set dst (ConvD2I src));
13871
13872 ins_cost(INSN_COST * 5);
13873 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13874
13875 ins_encode %{
13876 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13877 %}
13878
13879 ins_pipe(fp_d2i);
13880 %}
13881
13882 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13883 match(Set dst (ConvD2L src));
13884
13885 ins_cost(INSN_COST * 5);
13886 format %{ "fcvtzd $dst, $src \t// d2l" %}
13887
13888 ins_encode %{
13889 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13890 %}
13891
13892 ins_pipe(fp_d2l);
13893 %}
13894
13895 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13896 match(Set dst (ConvI2D src));
13897
13898 ins_cost(INSN_COST * 5);
13899 format %{ "scvtfwd $dst, $src \t// i2d" %}
13900
13901 ins_encode %{
13902 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13903 %}
13904
13905 ins_pipe(fp_i2d);
13906 %}
13907
13908 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13909 match(Set dst (ConvL2D src));
13910
13911 ins_cost(INSN_COST * 5);
13912 format %{ "scvtfd $dst, $src \t// l2d" %}
13913
13914 ins_encode %{
13915 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13916 %}
13917
13918 ins_pipe(fp_l2d);
13919 %}
13920
13921 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13922 %{
13923 match(Set dst (RoundD src));
13924 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13925 format %{ "java_round_double $dst,$src"%}
13926 ins_encode %{
13927 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13928 as_FloatRegister($ftmp$$reg));
13929 %}
13930 ins_pipe(pipe_slow);
13931 %}
13932
13933 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13934 %{
13935 match(Set dst (RoundF src));
13936 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13937 format %{ "java_round_float $dst,$src"%}
13938 ins_encode %{
13939 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13940 as_FloatRegister($ftmp$$reg));
13941 %}
13942 ins_pipe(pipe_slow);
13943 %}
13944
13945 // stack <-> reg and reg <-> reg shuffles with no conversion
13946
13947 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13948
13949 match(Set dst (MoveF2I src));
13950
13951 effect(DEF dst, USE src);
13952
13953 ins_cost(4 * INSN_COST);
13954
13955 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13956
13957 ins_encode %{
13958 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13959 %}
13960
13961 ins_pipe(iload_reg_reg);
13962
13963 %}
13964
13965 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13966
13967 match(Set dst (MoveI2F src));
13968
13969 effect(DEF dst, USE src);
13970
13971 ins_cost(4 * INSN_COST);
13972
13973 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13974
13975 ins_encode %{
13976 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13977 %}
13978
13979 ins_pipe(pipe_class_memory);
13980
13981 %}
13982
13983 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13984
13985 match(Set dst (MoveD2L src));
13986
13987 effect(DEF dst, USE src);
13988
13989 ins_cost(4 * INSN_COST);
13990
13991 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13992
13993 ins_encode %{
13994 __ ldr($dst$$Register, Address(sp, $src$$disp));
13995 %}
13996
13997 ins_pipe(iload_reg_reg);
13998
13999 %}
14000
14001 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14002
14003 match(Set dst (MoveL2D src));
14004
14005 effect(DEF dst, USE src);
14006
14007 ins_cost(4 * INSN_COST);
14008
14009 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14010
14011 ins_encode %{
14012 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14013 %}
14014
14015 ins_pipe(pipe_class_memory);
14016
14017 %}
14018
14019 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14020
14021 match(Set dst (MoveF2I src));
14022
14023 effect(DEF dst, USE src);
14024
14025 ins_cost(INSN_COST);
14026
14027 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14028
14029 ins_encode %{
14030 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14031 %}
14032
14033 ins_pipe(pipe_class_memory);
14034
14035 %}
14036
14037 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14038
14039 match(Set dst (MoveI2F src));
14040
14041 effect(DEF dst, USE src);
14042
14043 ins_cost(INSN_COST);
14044
14045 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14046
14047 ins_encode %{
14048 __ strw($src$$Register, Address(sp, $dst$$disp));
14049 %}
14050
14051 ins_pipe(istore_reg_reg);
14052
14053 %}
14054
14055 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14056
14057 match(Set dst (MoveD2L src));
14058
14059 effect(DEF dst, USE src);
14060
14061 ins_cost(INSN_COST);
14062
14063 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14064
14065 ins_encode %{
14066 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14067 %}
14068
14069 ins_pipe(pipe_class_memory);
14070
14071 %}
14072
14073 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14074
14075 match(Set dst (MoveL2D src));
14076
14077 effect(DEF dst, USE src);
14078
14079 ins_cost(INSN_COST);
14080
14081 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14082
14083 ins_encode %{
14084 __ str($src$$Register, Address(sp, $dst$$disp));
14085 %}
14086
14087 ins_pipe(istore_reg_reg);
14088
14089 %}
14090
14091 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14092
14093 match(Set dst (MoveF2I src));
14094
14095 effect(DEF dst, USE src);
14096
14097 ins_cost(INSN_COST);
14098
14099 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14100
14101 ins_encode %{
14102 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14103 %}
14104
14105 ins_pipe(fp_f2i);
14106
14107 %}
14108
14109 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14110
14111 match(Set dst (MoveI2F src));
14112
14113 effect(DEF dst, USE src);
14114
14115 ins_cost(INSN_COST);
14116
14117 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14118
14119 ins_encode %{
14120 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14121 %}
14122
14123 ins_pipe(fp_i2f);
14124
14125 %}
14126
14127 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14128
14129 match(Set dst (MoveD2L src));
14130
14131 effect(DEF dst, USE src);
14132
14133 ins_cost(INSN_COST);
14134
14135 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14136
14137 ins_encode %{
14138 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14139 %}
14140
14141 ins_pipe(fp_d2l);
14142
14143 %}
14144
14145 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14146
14147 match(Set dst (MoveL2D src));
14148
14149 effect(DEF dst, USE src);
14150
14151 ins_cost(INSN_COST);
14152
14153 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14154
14155 ins_encode %{
14156 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14157 %}
14158
14159 ins_pipe(fp_l2d);
14160
14161 %}
14162
14163 // ============================================================================
14164 // clearing of an array
14165
14166 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14167 %{
14168 match(Set dummy (ClearArray (Binary cnt base) zero));
14169 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14170
14171 ins_cost(4 * INSN_COST);
14172 format %{ "ClearArray $cnt, $base" %}
14173
14174 ins_encode %{
14175 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14176 if (tpc == nullptr) {
14177 ciEnv::current()->record_failure("CodeCache is full");
14178 return;
14179 }
14180 %}
14181
14182 ins_pipe(pipe_class_memory);
14183 %}
14184
14185 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14186 %{
14187 predicate(((ClearArrayNode*)n)->word_copy_only());
14188 match(Set dummy (ClearArray (Binary cnt base) val));
14189 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14190
14191 ins_cost(4 * INSN_COST);
14192 format %{ "ClearArray $cnt, $base, $val" %}
14193
14194 ins_encode %{
14195 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14196 %}
14197
14198 ins_pipe(pipe_class_memory);
14199 %}
14200
14201 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, immL0 zero, Universe dummy, rFlagsReg cr)
14202 %{
14203 predicate((uint64_t)n->in(2)->in(1)->get_long()
14204 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14205 && !((ClearArrayNode*)n)->word_copy_only());
14206 match(Set dummy (ClearArray (Binary cnt base) zero));
14207 effect(TEMP temp, USE_KILL base, KILL cr);
14208
14209 ins_cost(4 * INSN_COST);
14210 format %{ "ClearArray $cnt, $base" %}
14211
14212 ins_encode %{
14213 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14214 if (tpc == nullptr) {
14215 ciEnv::current()->record_failure("CodeCache is full");
14216 return;
14217 }
14218 %}
14219
14220 ins_pipe(pipe_class_memory);
14221 %}
14222
14223 // ============================================================================
14224 // Overflow Math Instructions
14225
14226 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14227 %{
14228 match(Set cr (OverflowAddI op1 op2));
14229
14230 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14231 ins_cost(INSN_COST);
14232 ins_encode %{
14233 __ cmnw($op1$$Register, $op2$$Register);
14234 %}
14235
14236 ins_pipe(icmp_reg_reg);
14237 %}
14238
14239 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14240 %{
14241 match(Set cr (OverflowAddI op1 op2));
14242
14243 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14244 ins_cost(INSN_COST);
14245 ins_encode %{
14246 __ cmnw($op1$$Register, $op2$$constant);
14247 %}
14248
14249 ins_pipe(icmp_reg_imm);
14250 %}
14251
14252 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14253 %{
14254 match(Set cr (OverflowAddL op1 op2));
14255
14256 format %{ "cmn $op1, $op2\t# overflow check long" %}
14257 ins_cost(INSN_COST);
14258 ins_encode %{
14259 __ cmn($op1$$Register, $op2$$Register);
14260 %}
14261
14262 ins_pipe(icmp_reg_reg);
14263 %}
14264
14265 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14266 %{
14267 match(Set cr (OverflowAddL op1 op2));
14268
14269 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14270 ins_cost(INSN_COST);
14271 ins_encode %{
14272 __ adds(zr, $op1$$Register, $op2$$constant);
14273 %}
14274
14275 ins_pipe(icmp_reg_imm);
14276 %}
14277
14278 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14279 %{
14280 match(Set cr (OverflowSubI op1 op2));
14281
14282 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14283 ins_cost(INSN_COST);
14284 ins_encode %{
14285 __ cmpw($op1$$Register, $op2$$Register);
14286 %}
14287
14288 ins_pipe(icmp_reg_reg);
14289 %}
14290
14291 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14292 %{
14293 match(Set cr (OverflowSubI op1 op2));
14294
14295 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14296 ins_cost(INSN_COST);
14297 ins_encode %{
14298 __ cmpw($op1$$Register, $op2$$constant);
14299 %}
14300
14301 ins_pipe(icmp_reg_imm);
14302 %}
14303
14304 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14305 %{
14306 match(Set cr (OverflowSubL op1 op2));
14307
14308 format %{ "cmp $op1, $op2\t# overflow check long" %}
14309 ins_cost(INSN_COST);
14310 ins_encode %{
14311 __ cmp($op1$$Register, $op2$$Register);
14312 %}
14313
14314 ins_pipe(icmp_reg_reg);
14315 %}
14316
14317 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14318 %{
14319 match(Set cr (OverflowSubL op1 op2));
14320
14321 format %{ "cmp $op1, $op2\t# overflow check long" %}
14322 ins_cost(INSN_COST);
14323 ins_encode %{
14324 __ subs(zr, $op1$$Register, $op2$$constant);
14325 %}
14326
14327 ins_pipe(icmp_reg_imm);
14328 %}
14329
14330 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14331 %{
14332 match(Set cr (OverflowSubI zero op1));
14333
14334 format %{ "cmpw zr, $op1\t# overflow check int" %}
14335 ins_cost(INSN_COST);
14336 ins_encode %{
14337 __ cmpw(zr, $op1$$Register);
14338 %}
14339
14340 ins_pipe(icmp_reg_imm);
14341 %}
14342
14343 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14344 %{
14345 match(Set cr (OverflowSubL zero op1));
14346
14347 format %{ "cmp zr, $op1\t# overflow check long" %}
14348 ins_cost(INSN_COST);
14349 ins_encode %{
14350 __ cmp(zr, $op1$$Register);
14351 %}
14352
14353 ins_pipe(icmp_reg_imm);
14354 %}
14355
14356 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14357 %{
14358 match(Set cr (OverflowMulI op1 op2));
14359
14360 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14361 "cmp rscratch1, rscratch1, sxtw\n\t"
14362 "movw rscratch1, #0x80000000\n\t"
14363 "cselw rscratch1, rscratch1, zr, NE\n\t"
14364 "cmpw rscratch1, #1" %}
14365 ins_cost(5 * INSN_COST);
14366 ins_encode %{
14367 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14368 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14369 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14370 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14371 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14372 %}
14373
14374 ins_pipe(pipe_slow);
14375 %}
14376
14377 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14378 %{
14379 match(If cmp (OverflowMulI op1 op2));
14380 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14381 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14382 effect(USE labl, KILL cr);
14383
14384 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14385 "cmp rscratch1, rscratch1, sxtw\n\t"
14386 "b$cmp $labl" %}
14387 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14388 ins_encode %{
14389 Label* L = $labl$$label;
14390 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14391 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14392 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14393 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14394 %}
14395
14396 ins_pipe(pipe_serial);
14397 %}
14398
14399 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14400 %{
14401 match(Set cr (OverflowMulL op1 op2));
14402
14403 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14404 "smulh rscratch2, $op1, $op2\n\t"
14405 "cmp rscratch2, rscratch1, ASR #63\n\t"
14406 "movw rscratch1, #0x80000000\n\t"
14407 "cselw rscratch1, rscratch1, zr, NE\n\t"
14408 "cmpw rscratch1, #1" %}
14409 ins_cost(6 * INSN_COST);
14410 ins_encode %{
14411 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14412 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14413 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14414 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14415 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14416 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14417 %}
14418
14419 ins_pipe(pipe_slow);
14420 %}
14421
14422 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14423 %{
14424 match(If cmp (OverflowMulL op1 op2));
14425 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14426 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14427 effect(USE labl, KILL cr);
14428
14429 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14430 "smulh rscratch2, $op1, $op2\n\t"
14431 "cmp rscratch2, rscratch1, ASR #63\n\t"
14432 "b$cmp $labl" %}
14433 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14434 ins_encode %{
14435 Label* L = $labl$$label;
14436 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14437 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14438 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14439 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14440 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14441 %}
14442
14443 ins_pipe(pipe_serial);
14444 %}
14445
14446 // ============================================================================
14447 // Compare Instructions
14448
14449 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14450 %{
14451 match(Set cr (CmpI op1 op2));
14452
14453 effect(DEF cr, USE op1, USE op2);
14454
14455 ins_cost(INSN_COST);
14456 format %{ "cmpw $op1, $op2" %}
14457
14458 ins_encode(aarch64_enc_cmpw(op1, op2));
14459
14460 ins_pipe(icmp_reg_reg);
14461 %}
14462
14463 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14464 %{
14465 match(Set cr (CmpI op1 zero));
14466
14467 effect(DEF cr, USE op1);
14468
14469 ins_cost(INSN_COST);
14470 format %{ "cmpw $op1, 0" %}
14471
14472 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14473
14474 ins_pipe(icmp_reg_imm);
14475 %}
14476
14477 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14478 %{
14479 match(Set cr (CmpI op1 op2));
14480
14481 effect(DEF cr, USE op1);
14482
14483 ins_cost(INSN_COST);
14484 format %{ "cmpw $op1, $op2" %}
14485
14486 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14487
14488 ins_pipe(icmp_reg_imm);
14489 %}
14490
14491 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14492 %{
14493 match(Set cr (CmpI op1 op2));
14494
14495 effect(DEF cr, USE op1);
14496
14497 ins_cost(INSN_COST * 2);
14498 format %{ "cmpw $op1, $op2" %}
14499
14500 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14501
14502 ins_pipe(icmp_reg_imm);
14503 %}
14504
14505 // Unsigned compare Instructions; really, same as signed compare
14506 // except it should only be used to feed an If or a CMovI which takes a
14507 // cmpOpU.
14508
14509 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14510 %{
14511 match(Set cr (CmpU op1 op2));
14512
14513 effect(DEF cr, USE op1, USE op2);
14514
14515 ins_cost(INSN_COST);
14516 format %{ "cmpw $op1, $op2\t# unsigned" %}
14517
14518 ins_encode(aarch64_enc_cmpw(op1, op2));
14519
14520 ins_pipe(icmp_reg_reg);
14521 %}
14522
14523 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14524 %{
14525 match(Set cr (CmpU op1 zero));
14526
14527 effect(DEF cr, USE op1);
14528
14529 ins_cost(INSN_COST);
14530 format %{ "cmpw $op1, #0\t# unsigned" %}
14531
14532 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14533
14534 ins_pipe(icmp_reg_imm);
14535 %}
14536
14537 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14538 %{
14539 match(Set cr (CmpU op1 op2));
14540
14541 effect(DEF cr, USE op1);
14542
14543 ins_cost(INSN_COST);
14544 format %{ "cmpw $op1, $op2\t# unsigned" %}
14545
14546 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14547
14548 ins_pipe(icmp_reg_imm);
14549 %}
14550
14551 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14552 %{
14553 match(Set cr (CmpU op1 op2));
14554
14555 effect(DEF cr, USE op1);
14556
14557 ins_cost(INSN_COST * 2);
14558 format %{ "cmpw $op1, $op2\t# unsigned" %}
14559
14560 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14561
14562 ins_pipe(icmp_reg_imm);
14563 %}
14564
14565 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14566 %{
14567 match(Set cr (CmpL op1 op2));
14568
14569 effect(DEF cr, USE op1, USE op2);
14570
14571 ins_cost(INSN_COST);
14572 format %{ "cmp $op1, $op2" %}
14573
14574 ins_encode(aarch64_enc_cmp(op1, op2));
14575
14576 ins_pipe(icmp_reg_reg);
14577 %}
14578
14579 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14580 %{
14581 match(Set cr (CmpL op1 zero));
14582
14583 effect(DEF cr, USE op1);
14584
14585 ins_cost(INSN_COST);
14586 format %{ "tst $op1" %}
14587
14588 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14589
14590 ins_pipe(icmp_reg_imm);
14591 %}
14592
14593 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14594 %{
14595 match(Set cr (CmpL op1 op2));
14596
14597 effect(DEF cr, USE op1);
14598
14599 ins_cost(INSN_COST);
14600 format %{ "cmp $op1, $op2" %}
14601
14602 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14603
14604 ins_pipe(icmp_reg_imm);
14605 %}
14606
14607 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14608 %{
14609 match(Set cr (CmpL op1 op2));
14610
14611 effect(DEF cr, USE op1);
14612
14613 ins_cost(INSN_COST * 2);
14614 format %{ "cmp $op1, $op2" %}
14615
14616 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14617
14618 ins_pipe(icmp_reg_imm);
14619 %}
14620
14621 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14622 %{
14623 match(Set cr (CmpUL op1 op2));
14624
14625 effect(DEF cr, USE op1, USE op2);
14626
14627 ins_cost(INSN_COST);
14628 format %{ "cmp $op1, $op2" %}
14629
14630 ins_encode(aarch64_enc_cmp(op1, op2));
14631
14632 ins_pipe(icmp_reg_reg);
14633 %}
14634
14635 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14636 %{
14637 match(Set cr (CmpUL op1 zero));
14638
14639 effect(DEF cr, USE op1);
14640
14641 ins_cost(INSN_COST);
14642 format %{ "tst $op1" %}
14643
14644 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14645
14646 ins_pipe(icmp_reg_imm);
14647 %}
14648
14649 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14650 %{
14651 match(Set cr (CmpUL op1 op2));
14652
14653 effect(DEF cr, USE op1);
14654
14655 ins_cost(INSN_COST);
14656 format %{ "cmp $op1, $op2" %}
14657
14658 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14659
14660 ins_pipe(icmp_reg_imm);
14661 %}
14662
14663 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14664 %{
14665 match(Set cr (CmpUL op1 op2));
14666
14667 effect(DEF cr, USE op1);
14668
14669 ins_cost(INSN_COST * 2);
14670 format %{ "cmp $op1, $op2" %}
14671
14672 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14673
14674 ins_pipe(icmp_reg_imm);
14675 %}
14676
14677 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14678 %{
14679 match(Set cr (CmpP op1 op2));
14680
14681 effect(DEF cr, USE op1, USE op2);
14682
14683 ins_cost(INSN_COST);
14684 format %{ "cmp $op1, $op2\t // ptr" %}
14685
14686 ins_encode(aarch64_enc_cmpp(op1, op2));
14687
14688 ins_pipe(icmp_reg_reg);
14689 %}
14690
14691 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14692 %{
14693 match(Set cr (CmpN op1 op2));
14694
14695 effect(DEF cr, USE op1, USE op2);
14696
14697 ins_cost(INSN_COST);
14698 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14699
14700 ins_encode(aarch64_enc_cmpn(op1, op2));
14701
14702 ins_pipe(icmp_reg_reg);
14703 %}
14704
14705 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14706 %{
14707 match(Set cr (CmpP op1 zero));
14708
14709 effect(DEF cr, USE op1, USE zero);
14710
14711 ins_cost(INSN_COST);
14712 format %{ "cmp $op1, 0\t // ptr" %}
14713
14714 ins_encode(aarch64_enc_testp(op1));
14715
14716 ins_pipe(icmp_reg_imm);
14717 %}
14718
14719 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14720 %{
14721 match(Set cr (CmpN op1 zero));
14722
14723 effect(DEF cr, USE op1, USE zero);
14724
14725 ins_cost(INSN_COST);
14726 format %{ "cmp $op1, 0\t // compressed ptr" %}
14727
14728 ins_encode(aarch64_enc_testn(op1));
14729
14730 ins_pipe(icmp_reg_imm);
14731 %}
14732
14733 // FP comparisons
14734 //
14735 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14736 // using normal cmpOp. See declaration of rFlagsReg for details.
14737
14738 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14739 %{
14740 match(Set cr (CmpF src1 src2));
14741
14742 ins_cost(3 * INSN_COST);
14743 format %{ "fcmps $src1, $src2" %}
14744
14745 ins_encode %{
14746 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14747 %}
14748
14749 ins_pipe(pipe_class_compare);
14750 %}
14751
14752 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14753 %{
14754 match(Set cr (CmpF src1 src2));
14755
14756 ins_cost(3 * INSN_COST);
14757 format %{ "fcmps $src1, 0.0" %}
14758
14759 ins_encode %{
14760 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14761 %}
14762
14763 ins_pipe(pipe_class_compare);
14764 %}
14765 // FROM HERE
14766
14767 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14768 %{
14769 match(Set cr (CmpD src1 src2));
14770
14771 ins_cost(3 * INSN_COST);
14772 format %{ "fcmpd $src1, $src2" %}
14773
14774 ins_encode %{
14775 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14776 %}
14777
14778 ins_pipe(pipe_class_compare);
14779 %}
14780
14781 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14782 %{
14783 match(Set cr (CmpD src1 src2));
14784
14785 ins_cost(3 * INSN_COST);
14786 format %{ "fcmpd $src1, 0.0" %}
14787
14788 ins_encode %{
14789 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14790 %}
14791
14792 ins_pipe(pipe_class_compare);
14793 %}
14794
14795 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14796 %{
14797 match(Set dst (CmpF3 src1 src2));
14798 effect(KILL cr);
14799
14800 ins_cost(5 * INSN_COST);
14801 format %{ "fcmps $src1, $src2\n\t"
14802 "csinvw($dst, zr, zr, eq\n\t"
14803 "csnegw($dst, $dst, $dst, lt)"
14804 %}
14805
14806 ins_encode %{
14807 Label done;
14808 FloatRegister s1 = as_FloatRegister($src1$$reg);
14809 FloatRegister s2 = as_FloatRegister($src2$$reg);
14810 Register d = as_Register($dst$$reg);
14811 __ fcmps(s1, s2);
14812 // installs 0 if EQ else -1
14813 __ csinvw(d, zr, zr, Assembler::EQ);
14814 // keeps -1 if less or unordered else installs 1
14815 __ csnegw(d, d, d, Assembler::LT);
14816 __ bind(done);
14817 %}
14818
14819 ins_pipe(pipe_class_default);
14820
14821 %}
14822
14823 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14824 %{
14825 match(Set dst (CmpD3 src1 src2));
14826 effect(KILL cr);
14827
14828 ins_cost(5 * INSN_COST);
14829 format %{ "fcmpd $src1, $src2\n\t"
14830 "csinvw($dst, zr, zr, eq\n\t"
14831 "csnegw($dst, $dst, $dst, lt)"
14832 %}
14833
14834 ins_encode %{
14835 Label done;
14836 FloatRegister s1 = as_FloatRegister($src1$$reg);
14837 FloatRegister s2 = as_FloatRegister($src2$$reg);
14838 Register d = as_Register($dst$$reg);
14839 __ fcmpd(s1, s2);
14840 // installs 0 if EQ else -1
14841 __ csinvw(d, zr, zr, Assembler::EQ);
14842 // keeps -1 if less or unordered else installs 1
14843 __ csnegw(d, d, d, Assembler::LT);
14844 __ bind(done);
14845 %}
14846 ins_pipe(pipe_class_default);
14847
14848 %}
14849
14850 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14851 %{
14852 match(Set dst (CmpF3 src1 zero));
14853 effect(KILL cr);
14854
14855 ins_cost(5 * INSN_COST);
14856 format %{ "fcmps $src1, 0.0\n\t"
14857 "csinvw($dst, zr, zr, eq\n\t"
14858 "csnegw($dst, $dst, $dst, lt)"
14859 %}
14860
14861 ins_encode %{
14862 Label done;
14863 FloatRegister s1 = as_FloatRegister($src1$$reg);
14864 Register d = as_Register($dst$$reg);
14865 __ fcmps(s1, 0.0);
14866 // installs 0 if EQ else -1
14867 __ csinvw(d, zr, zr, Assembler::EQ);
14868 // keeps -1 if less or unordered else installs 1
14869 __ csnegw(d, d, d, Assembler::LT);
14870 __ bind(done);
14871 %}
14872
14873 ins_pipe(pipe_class_default);
14874
14875 %}
14876
14877 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14878 %{
14879 match(Set dst (CmpD3 src1 zero));
14880 effect(KILL cr);
14881
14882 ins_cost(5 * INSN_COST);
14883 format %{ "fcmpd $src1, 0.0\n\t"
14884 "csinvw($dst, zr, zr, eq\n\t"
14885 "csnegw($dst, $dst, $dst, lt)"
14886 %}
14887
14888 ins_encode %{
14889 Label done;
14890 FloatRegister s1 = as_FloatRegister($src1$$reg);
14891 Register d = as_Register($dst$$reg);
14892 __ fcmpd(s1, 0.0);
14893 // installs 0 if EQ else -1
14894 __ csinvw(d, zr, zr, Assembler::EQ);
14895 // keeps -1 if less or unordered else installs 1
14896 __ csnegw(d, d, d, Assembler::LT);
14897 __ bind(done);
14898 %}
14899 ins_pipe(pipe_class_default);
14900
14901 %}
14902
14903 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14904 %{
14905 match(Set dst (CmpLTMask p q));
14906 effect(KILL cr);
14907
14908 ins_cost(3 * INSN_COST);
14909
14910 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14911 "csetw $dst, lt\n\t"
14912 "subw $dst, zr, $dst"
14913 %}
14914
14915 ins_encode %{
14916 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14917 __ csetw(as_Register($dst$$reg), Assembler::LT);
14918 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14919 %}
14920
14921 ins_pipe(ialu_reg_reg);
14922 %}
14923
14924 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14925 %{
14926 match(Set dst (CmpLTMask src zero));
14927 effect(KILL cr);
14928
14929 ins_cost(INSN_COST);
14930
14931 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14932
14933 ins_encode %{
14934 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14935 %}
14936
14937 ins_pipe(ialu_reg_shift);
14938 %}
14939
14940 // ============================================================================
14941 // Max and Min
14942
14943 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14944
14945 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14946 %{
14947 effect(DEF cr, USE src);
14948 ins_cost(INSN_COST);
14949 format %{ "cmpw $src, 0" %}
14950
14951 ins_encode %{
14952 __ cmpw($src$$Register, 0);
14953 %}
14954 ins_pipe(icmp_reg_imm);
14955 %}
14956
14957 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14958 %{
14959 match(Set dst (MinI src1 src2));
14960 ins_cost(INSN_COST * 3);
14961
14962 expand %{
14963 rFlagsReg cr;
14964 compI_reg_reg(cr, src1, src2);
14965 cmovI_reg_reg_lt(dst, src1, src2, cr);
14966 %}
14967 %}
14968
14969 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14970 %{
14971 match(Set dst (MaxI src1 src2));
14972 ins_cost(INSN_COST * 3);
14973
14974 expand %{
14975 rFlagsReg cr;
14976 compI_reg_reg(cr, src1, src2);
14977 cmovI_reg_reg_gt(dst, src1, src2, cr);
14978 %}
14979 %}
14980
14981
14982 // ============================================================================
14983 // Branch Instructions
14984
14985 // Direct Branch.
14986 instruct branch(label lbl)
14987 %{
14988 match(Goto);
14989
14990 effect(USE lbl);
14991
14992 ins_cost(BRANCH_COST);
14993 format %{ "b $lbl" %}
14994
14995 ins_encode(aarch64_enc_b(lbl));
14996
14997 ins_pipe(pipe_branch);
14998 %}
14999
15000 // Conditional Near Branch
15001 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15002 %{
15003 // Same match rule as `branchConFar'.
15004 match(If cmp cr);
15005
15006 effect(USE lbl);
15007
15008 ins_cost(BRANCH_COST);
15009 // If set to 1 this indicates that the current instruction is a
15010 // short variant of a long branch. This avoids using this
15011 // instruction in first-pass matching. It will then only be used in
15012 // the `Shorten_branches' pass.
15013 // ins_short_branch(1);
15014 format %{ "b$cmp $lbl" %}
15015
15016 ins_encode(aarch64_enc_br_con(cmp, lbl));
15017
15018 ins_pipe(pipe_branch_cond);
15019 %}
15020
15021 // Conditional Near Branch Unsigned
15022 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15023 %{
15024 // Same match rule as `branchConFar'.
15025 match(If cmp cr);
15026
15027 effect(USE lbl);
15028
15029 ins_cost(BRANCH_COST);
15030 // If set to 1 this indicates that the current instruction is a
15031 // short variant of a long branch. This avoids using this
15032 // instruction in first-pass matching. It will then only be used in
15033 // the `Shorten_branches' pass.
15034 // ins_short_branch(1);
15035 format %{ "b$cmp $lbl\t# unsigned" %}
15036
15037 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15038
15039 ins_pipe(pipe_branch_cond);
15040 %}
15041
15042 // Make use of CBZ and CBNZ. These instructions, as well as being
15043 // shorter than (cmp; branch), have the additional benefit of not
15044 // killing the flags.
15045
15046 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15047 match(If cmp (CmpI op1 op2));
15048 effect(USE labl);
15049
15050 ins_cost(BRANCH_COST);
15051 format %{ "cbw$cmp $op1, $labl" %}
15052 ins_encode %{
15053 Label* L = $labl$$label;
15054 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15055 if (cond == Assembler::EQ)
15056 __ cbzw($op1$$Register, *L);
15057 else
15058 __ cbnzw($op1$$Register, *L);
15059 %}
15060 ins_pipe(pipe_cmp_branch);
15061 %}
15062
15063 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15064 match(If cmp (CmpL op1 op2));
15065 effect(USE labl);
15066
15067 ins_cost(BRANCH_COST);
15068 format %{ "cb$cmp $op1, $labl" %}
15069 ins_encode %{
15070 Label* L = $labl$$label;
15071 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15072 if (cond == Assembler::EQ)
15073 __ cbz($op1$$Register, *L);
15074 else
15075 __ cbnz($op1$$Register, *L);
15076 %}
15077 ins_pipe(pipe_cmp_branch);
15078 %}
15079
15080 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15081 match(If cmp (CmpP op1 op2));
15082 effect(USE labl);
15083
15084 ins_cost(BRANCH_COST);
15085 format %{ "cb$cmp $op1, $labl" %}
15086 ins_encode %{
15087 Label* L = $labl$$label;
15088 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15089 if (cond == Assembler::EQ)
15090 __ cbz($op1$$Register, *L);
15091 else
15092 __ cbnz($op1$$Register, *L);
15093 %}
15094 ins_pipe(pipe_cmp_branch);
15095 %}
15096
15097 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15098 match(If cmp (CmpN op1 op2));
15099 effect(USE labl);
15100
15101 ins_cost(BRANCH_COST);
15102 format %{ "cbw$cmp $op1, $labl" %}
15103 ins_encode %{
15104 Label* L = $labl$$label;
15105 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15106 if (cond == Assembler::EQ)
15107 __ cbzw($op1$$Register, *L);
15108 else
15109 __ cbnzw($op1$$Register, *L);
15110 %}
15111 ins_pipe(pipe_cmp_branch);
15112 %}
15113
15114 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15115 match(If cmp (CmpP (DecodeN oop) zero));
15116 effect(USE labl);
15117
15118 ins_cost(BRANCH_COST);
15119 format %{ "cb$cmp $oop, $labl" %}
15120 ins_encode %{
15121 Label* L = $labl$$label;
15122 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15123 if (cond == Assembler::EQ)
15124 __ cbzw($oop$$Register, *L);
15125 else
15126 __ cbnzw($oop$$Register, *L);
15127 %}
15128 ins_pipe(pipe_cmp_branch);
15129 %}
15130
15131 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15132 match(If cmp (CmpU op1 op2));
15133 effect(USE labl);
15134
15135 ins_cost(BRANCH_COST);
15136 format %{ "cbw$cmp $op1, $labl" %}
15137 ins_encode %{
15138 Label* L = $labl$$label;
15139 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15140 if (cond == Assembler::EQ || cond == Assembler::LS) {
15141 __ cbzw($op1$$Register, *L);
15142 } else {
15143 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15144 __ cbnzw($op1$$Register, *L);
15145 }
15146 %}
15147 ins_pipe(pipe_cmp_branch);
15148 %}
15149
15150 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15151 match(If cmp (CmpUL op1 op2));
15152 effect(USE labl);
15153
15154 ins_cost(BRANCH_COST);
15155 format %{ "cb$cmp $op1, $labl" %}
15156 ins_encode %{
15157 Label* L = $labl$$label;
15158 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15159 if (cond == Assembler::EQ || cond == Assembler::LS) {
15160 __ cbz($op1$$Register, *L);
15161 } else {
15162 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15163 __ cbnz($op1$$Register, *L);
15164 }
15165 %}
15166 ins_pipe(pipe_cmp_branch);
15167 %}
15168
15169 // Test bit and Branch
15170
15171 // Patterns for short (< 32KiB) variants
15172 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15173 match(If cmp (CmpL op1 op2));
15174 effect(USE labl);
15175
15176 ins_cost(BRANCH_COST);
15177 format %{ "cb$cmp $op1, $labl # long" %}
15178 ins_encode %{
15179 Label* L = $labl$$label;
15180 Assembler::Condition cond =
15181 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15182 __ tbr(cond, $op1$$Register, 63, *L);
15183 %}
15184 ins_pipe(pipe_cmp_branch);
15185 ins_short_branch(1);
15186 %}
15187
15188 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15189 match(If cmp (CmpI op1 op2));
15190 effect(USE labl);
15191
15192 ins_cost(BRANCH_COST);
15193 format %{ "cb$cmp $op1, $labl # int" %}
15194 ins_encode %{
15195 Label* L = $labl$$label;
15196 Assembler::Condition cond =
15197 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15198 __ tbr(cond, $op1$$Register, 31, *L);
15199 %}
15200 ins_pipe(pipe_cmp_branch);
15201 ins_short_branch(1);
15202 %}
15203
15204 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15205 match(If cmp (CmpL (AndL op1 op2) op3));
15206 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15207 effect(USE labl);
15208
15209 ins_cost(BRANCH_COST);
15210 format %{ "tb$cmp $op1, $op2, $labl" %}
15211 ins_encode %{
15212 Label* L = $labl$$label;
15213 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15214 int bit = exact_log2_long($op2$$constant);
15215 __ tbr(cond, $op1$$Register, bit, *L);
15216 %}
15217 ins_pipe(pipe_cmp_branch);
15218 ins_short_branch(1);
15219 %}
15220
15221 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15222 match(If cmp (CmpI (AndI op1 op2) op3));
15223 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15224 effect(USE labl);
15225
15226 ins_cost(BRANCH_COST);
15227 format %{ "tb$cmp $op1, $op2, $labl" %}
15228 ins_encode %{
15229 Label* L = $labl$$label;
15230 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15231 int bit = exact_log2((juint)$op2$$constant);
15232 __ tbr(cond, $op1$$Register, bit, *L);
15233 %}
15234 ins_pipe(pipe_cmp_branch);
15235 ins_short_branch(1);
15236 %}
15237
15238 // And far variants
15239 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15240 match(If cmp (CmpL op1 op2));
15241 effect(USE labl);
15242
15243 ins_cost(BRANCH_COST);
15244 format %{ "cb$cmp $op1, $labl # long" %}
15245 ins_encode %{
15246 Label* L = $labl$$label;
15247 Assembler::Condition cond =
15248 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15249 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15250 %}
15251 ins_pipe(pipe_cmp_branch);
15252 %}
15253
15254 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15255 match(If cmp (CmpI op1 op2));
15256 effect(USE labl);
15257
15258 ins_cost(BRANCH_COST);
15259 format %{ "cb$cmp $op1, $labl # int" %}
15260 ins_encode %{
15261 Label* L = $labl$$label;
15262 Assembler::Condition cond =
15263 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15264 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15265 %}
15266 ins_pipe(pipe_cmp_branch);
15267 %}
15268
15269 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15270 match(If cmp (CmpL (AndL op1 op2) op3));
15271 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15272 effect(USE labl);
15273
15274 ins_cost(BRANCH_COST);
15275 format %{ "tb$cmp $op1, $op2, $labl" %}
15276 ins_encode %{
15277 Label* L = $labl$$label;
15278 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15279 int bit = exact_log2_long($op2$$constant);
15280 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15281 %}
15282 ins_pipe(pipe_cmp_branch);
15283 %}
15284
15285 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15286 match(If cmp (CmpI (AndI op1 op2) op3));
15287 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15288 effect(USE labl);
15289
15290 ins_cost(BRANCH_COST);
15291 format %{ "tb$cmp $op1, $op2, $labl" %}
15292 ins_encode %{
15293 Label* L = $labl$$label;
15294 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15295 int bit = exact_log2((juint)$op2$$constant);
15296 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15297 %}
15298 ins_pipe(pipe_cmp_branch);
15299 %}
15300
15301 // Test bits
15302
15303 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15304 match(Set cr (CmpL (AndL op1 op2) op3));
15305 predicate(Assembler::operand_valid_for_logical_immediate
15306 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15307
15308 ins_cost(INSN_COST);
15309 format %{ "tst $op1, $op2 # long" %}
15310 ins_encode %{
15311 __ tst($op1$$Register, $op2$$constant);
15312 %}
15313 ins_pipe(ialu_reg_reg);
15314 %}
15315
15316 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15317 match(Set cr (CmpI (AndI op1 op2) op3));
15318 predicate(Assembler::operand_valid_for_logical_immediate
15319 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15320
15321 ins_cost(INSN_COST);
15322 format %{ "tst $op1, $op2 # int" %}
15323 ins_encode %{
15324 __ tstw($op1$$Register, $op2$$constant);
15325 %}
15326 ins_pipe(ialu_reg_reg);
15327 %}
15328
15329 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15330 match(Set cr (CmpL (AndL op1 op2) op3));
15331
15332 ins_cost(INSN_COST);
15333 format %{ "tst $op1, $op2 # long" %}
15334 ins_encode %{
15335 __ tst($op1$$Register, $op2$$Register);
15336 %}
15337 ins_pipe(ialu_reg_reg);
15338 %}
15339
15340 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15341 match(Set cr (CmpI (AndI op1 op2) op3));
15342
15343 ins_cost(INSN_COST);
15344 format %{ "tstw $op1, $op2 # int" %}
15345 ins_encode %{
15346 __ tstw($op1$$Register, $op2$$Register);
15347 %}
15348 ins_pipe(ialu_reg_reg);
15349 %}
15350
15351
15352 // Conditional Far Branch
15353 // Conditional Far Branch Unsigned
15354 // TODO: fixme
15355
15356 // counted loop end branch near
15357 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15358 %{
15359 match(CountedLoopEnd cmp cr);
15360
15361 effect(USE lbl);
15362
15363 ins_cost(BRANCH_COST);
15364 // short variant.
15365 // ins_short_branch(1);
15366 format %{ "b$cmp $lbl \t// counted loop end" %}
15367
15368 ins_encode(aarch64_enc_br_con(cmp, lbl));
15369
15370 ins_pipe(pipe_branch);
15371 %}
15372
15373 // counted loop end branch far
15374 // TODO: fixme
15375
15376 // ============================================================================
15377 // inlined locking and unlocking
15378
15379 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15380 %{
15381 match(Set cr (FastLock object box));
15382 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15383
15384 ins_cost(5 * INSN_COST);
15385 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15386
15387 ins_encode %{
15388 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15389 %}
15390
15391 ins_pipe(pipe_serial);
15392 %}
15393
15394 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15395 %{
15396 match(Set cr (FastUnlock object box));
15397 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15398
15399 ins_cost(5 * INSN_COST);
15400 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15401
15402 ins_encode %{
15403 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15404 %}
15405
15406 ins_pipe(pipe_serial);
15407 %}
15408
15409 // ============================================================================
15410 // Safepoint Instructions
15411
15412 // TODO
15413 // provide a near and far version of this code
15414
15415 instruct safePoint(rFlagsReg cr, iRegP poll)
15416 %{
15417 match(SafePoint poll);
15418 effect(KILL cr);
15419
15420 format %{
15421 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15422 %}
15423 ins_encode %{
15424 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15425 %}
15426 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15427 %}
15428
15429
15430 // ============================================================================
15431 // Procedure Call/Return Instructions
15432
15433 // Call Java Static Instruction
15434
15435 instruct CallStaticJavaDirect(method meth)
15436 %{
15437 match(CallStaticJava);
15438
15439 effect(USE meth);
15440
15441 ins_cost(CALL_COST);
15442
15443 format %{ "call,static $meth \t// ==> " %}
15444
15445 ins_encode(aarch64_enc_java_static_call(meth),
15446 aarch64_enc_call_epilog);
15447
15448 ins_pipe(pipe_class_call);
15449 %}
15450
15451 // TO HERE
15452
15453 // Call Java Dynamic Instruction
15454 instruct CallDynamicJavaDirect(method meth)
15455 %{
15456 match(CallDynamicJava);
15457
15458 effect(USE meth);
15459
15460 ins_cost(CALL_COST);
15461
15462 format %{ "CALL,dynamic $meth \t// ==> " %}
15463
15464 ins_encode(aarch64_enc_java_dynamic_call(meth),
15465 aarch64_enc_call_epilog);
15466
15467 ins_pipe(pipe_class_call);
15468 %}
15469
15470 // Call Runtime Instruction
15471
15472 instruct CallRuntimeDirect(method meth)
15473 %{
15474 match(CallRuntime);
15475
15476 effect(USE meth);
15477
15478 ins_cost(CALL_COST);
15479
15480 format %{ "CALL, runtime $meth" %}
15481
15482 ins_encode( aarch64_enc_java_to_runtime(meth) );
15483
15484 ins_pipe(pipe_class_call);
15485 %}
15486
15487 // Call Runtime Instruction
15488
15489 instruct CallLeafDirect(method meth)
15490 %{
15491 match(CallLeaf);
15492
15493 effect(USE meth);
15494
15495 ins_cost(CALL_COST);
15496
15497 format %{ "CALL, runtime leaf $meth" %}
15498
15499 ins_encode( aarch64_enc_java_to_runtime(meth) );
15500
15501 ins_pipe(pipe_class_call);
15502 %}
15503
15504 // Call Runtime Instruction without safepoint and with vector arguments
15505 instruct CallLeafDirectVector(method meth)
15506 %{
15507 match(CallLeafVector);
15508
15509 effect(USE meth);
15510
15511 ins_cost(CALL_COST);
15512
15513 format %{ "CALL, runtime leaf vector $meth" %}
15514
15515 ins_encode(aarch64_enc_java_to_runtime(meth));
15516
15517 ins_pipe(pipe_class_call);
15518 %}
15519
15520 // Call Runtime Instruction
15521
15522 // entry point is null, target holds the address to call
15523 instruct CallLeafNoFPIndirect(iRegP target)
15524 %{
15525 predicate(n->as_Call()->entry_point() == nullptr);
15526
15527 match(CallLeafNoFP target);
15528
15529 ins_cost(CALL_COST);
15530
15531 format %{ "CALL, runtime leaf nofp indirect $target" %}
15532
15533 ins_encode %{
15534 __ blr($target$$Register);
15535 %}
15536
15537 ins_pipe(pipe_class_call);
15538 %}
15539
15540 instruct CallLeafNoFPDirect(method meth)
15541 %{
15542 predicate(n->as_Call()->entry_point() != nullptr);
15543
15544 match(CallLeafNoFP);
15545
15546 effect(USE meth);
15547
15548 ins_cost(CALL_COST);
15549
15550 format %{ "CALL, runtime leaf nofp $meth" %}
15551
15552 ins_encode( aarch64_enc_java_to_runtime(meth) );
15553
15554 ins_pipe(pipe_class_call);
15555 %}
15556
15557 // Tail Call; Jump from runtime stub to Java code.
15558 // Also known as an 'interprocedural jump'.
15559 // Target of jump will eventually return to caller.
15560 // TailJump below removes the return address.
15561 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15562 // emitted just above the TailCall which has reset rfp to the caller state.
15563 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15564 %{
15565 match(TailCall jump_target method_ptr);
15566
15567 ins_cost(CALL_COST);
15568
15569 format %{ "br $jump_target\t# $method_ptr holds method" %}
15570
15571 ins_encode(aarch64_enc_tail_call(jump_target));
15572
15573 ins_pipe(pipe_class_call);
15574 %}
15575
15576 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15577 %{
15578 match(TailJump jump_target ex_oop);
15579
15580 ins_cost(CALL_COST);
15581
15582 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15583
15584 ins_encode(aarch64_enc_tail_jmp(jump_target));
15585
15586 ins_pipe(pipe_class_call);
15587 %}
15588
15589 // Forward exception.
15590 instruct ForwardExceptionjmp()
15591 %{
15592 match(ForwardException);
15593 ins_cost(CALL_COST);
15594
15595 format %{ "b forward_exception_stub" %}
15596 ins_encode %{
15597 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15598 %}
15599 ins_pipe(pipe_class_call);
15600 %}
15601
15602 // Create exception oop: created by stack-crawling runtime code.
15603 // Created exception is now available to this handler, and is setup
15604 // just prior to jumping to this handler. No code emitted.
15605 // TODO check
15606 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15607 instruct CreateException(iRegP_R0 ex_oop)
15608 %{
15609 match(Set ex_oop (CreateEx));
15610
15611 format %{ " -- \t// exception oop; no code emitted" %}
15612
15613 size(0);
15614
15615 ins_encode( /*empty*/ );
15616
15617 ins_pipe(pipe_class_empty);
15618 %}
15619
15620 // Rethrow exception: The exception oop will come in the first
15621 // argument position. Then JUMP (not call) to the rethrow stub code.
15622 instruct RethrowException() %{
15623 match(Rethrow);
15624 ins_cost(CALL_COST);
15625
15626 format %{ "b rethrow_stub" %}
15627
15628 ins_encode( aarch64_enc_rethrow() );
15629
15630 ins_pipe(pipe_class_call);
15631 %}
15632
15633
15634 // Return Instruction
15635 // epilog node loads ret address into lr as part of frame pop
15636 instruct Ret()
15637 %{
15638 match(Return);
15639
15640 format %{ "ret\t// return register" %}
15641
15642 ins_encode( aarch64_enc_ret() );
15643
15644 ins_pipe(pipe_branch);
15645 %}
15646
15647 // Die now.
15648 instruct ShouldNotReachHere() %{
15649 match(Halt);
15650
15651 ins_cost(CALL_COST);
15652 format %{ "ShouldNotReachHere" %}
15653
15654 ins_encode %{
15655 if (is_reachable()) {
15656 const char* str = __ code_string(_halt_reason);
15657 __ stop(str);
15658 }
15659 %}
15660
15661 ins_pipe(pipe_class_default);
15662 %}
15663
15664 // ============================================================================
15665 // Partial Subtype Check
15666 //
15667 // superklass array for an instance of the superklass. Set a hidden
15668 // internal cache on a hit (cache is checked with exposed code in
15669 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15670 // encoding ALSO sets flags.
15671
15672 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15673 %{
15674 match(Set result (PartialSubtypeCheck sub super));
15675 predicate(!UseSecondarySupersTable);
15676 effect(KILL cr, KILL temp);
15677
15678 ins_cost(20 * INSN_COST); // slightly larger than the next version
15679 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15680
15681 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15682
15683 opcode(0x1); // Force zero of result reg on hit
15684
15685 ins_pipe(pipe_class_memory);
15686 %}
15687
15688 // Two versions of partialSubtypeCheck, both used when we need to
15689 // search for a super class in the secondary supers array. The first
15690 // is used when we don't know _a priori_ the class being searched
15691 // for. The second, far more common, is used when we do know: this is
15692 // used for instanceof, checkcast, and any case where C2 can determine
15693 // it by constant propagation.
15694
15695 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15696 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15697 rFlagsReg cr)
15698 %{
15699 match(Set result (PartialSubtypeCheck sub super));
15700 predicate(UseSecondarySupersTable);
15701 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15702
15703 ins_cost(10 * INSN_COST); // slightly larger than the next version
15704 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15705
15706 ins_encode %{
15707 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15708 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15709 $vtemp$$FloatRegister,
15710 $result$$Register, /*L_success*/nullptr);
15711 %}
15712
15713 ins_pipe(pipe_class_memory);
15714 %}
15715
15716 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15717 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15718 rFlagsReg cr)
15719 %{
15720 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15721 predicate(UseSecondarySupersTable);
15722 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15723
15724 ins_cost(5 * INSN_COST); // smaller than the next version
15725 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15726
15727 ins_encode %{
15728 bool success = false;
15729 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15730 if (InlineSecondarySupersTest) {
15731 success =
15732 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15733 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15734 $vtemp$$FloatRegister,
15735 $result$$Register,
15736 super_klass_slot);
15737 } else {
15738 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15739 success = (call != nullptr);
15740 }
15741 if (!success) {
15742 ciEnv::current()->record_failure("CodeCache is full");
15743 return;
15744 }
15745 %}
15746
15747 ins_pipe(pipe_class_memory);
15748 %}
15749
15750 // Intrisics for String.compareTo()
15751
15752 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15753 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15754 %{
15755 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15756 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15757 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15758
15759 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15760 ins_encode %{
15761 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15762 __ string_compare($str1$$Register, $str2$$Register,
15763 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15764 $tmp1$$Register, $tmp2$$Register,
15765 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15766 %}
15767 ins_pipe(pipe_class_memory);
15768 %}
15769
15770 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15771 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15772 %{
15773 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15774 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15775 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15776
15777 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15778 ins_encode %{
15779 __ string_compare($str1$$Register, $str2$$Register,
15780 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15781 $tmp1$$Register, $tmp2$$Register,
15782 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15783 %}
15784 ins_pipe(pipe_class_memory);
15785 %}
15786
15787 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15788 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15789 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15790 %{
15791 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15792 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15793 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15794 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15795
15796 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15797 ins_encode %{
15798 __ string_compare($str1$$Register, $str2$$Register,
15799 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15800 $tmp1$$Register, $tmp2$$Register,
15801 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15802 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15803 %}
15804 ins_pipe(pipe_class_memory);
15805 %}
15806
15807 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15808 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15809 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15810 %{
15811 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15812 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15813 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15814 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15815
15816 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15817 ins_encode %{
15818 __ string_compare($str1$$Register, $str2$$Register,
15819 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15820 $tmp1$$Register, $tmp2$$Register,
15821 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15822 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15823 %}
15824 ins_pipe(pipe_class_memory);
15825 %}
15826
15827 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15828 // these string_compare variants as NEON register type for convenience so that the prototype of
15829 // string_compare can be shared with all variants.
15830
15831 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15832 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15833 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15834 pRegGov_P1 pgtmp2, rFlagsReg cr)
15835 %{
15836 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15837 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15838 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15839 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15840
15841 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15842 ins_encode %{
15843 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15844 __ string_compare($str1$$Register, $str2$$Register,
15845 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15846 $tmp1$$Register, $tmp2$$Register,
15847 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15848 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15849 StrIntrinsicNode::LL);
15850 %}
15851 ins_pipe(pipe_class_memory);
15852 %}
15853
15854 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15855 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15856 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15857 pRegGov_P1 pgtmp2, rFlagsReg cr)
15858 %{
15859 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15860 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15861 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15862 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15863
15864 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15865 ins_encode %{
15866 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15867 __ string_compare($str1$$Register, $str2$$Register,
15868 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15869 $tmp1$$Register, $tmp2$$Register,
15870 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15871 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15872 StrIntrinsicNode::LU);
15873 %}
15874 ins_pipe(pipe_class_memory);
15875 %}
15876
15877 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15878 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15879 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15880 pRegGov_P1 pgtmp2, rFlagsReg cr)
15881 %{
15882 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15883 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15884 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15885 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15886
15887 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15888 ins_encode %{
15889 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15890 __ string_compare($str1$$Register, $str2$$Register,
15891 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15892 $tmp1$$Register, $tmp2$$Register,
15893 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15894 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15895 StrIntrinsicNode::UL);
15896 %}
15897 ins_pipe(pipe_class_memory);
15898 %}
15899
15900 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15901 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15902 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15903 pRegGov_P1 pgtmp2, rFlagsReg cr)
15904 %{
15905 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15906 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15907 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15908 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15909
15910 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15911 ins_encode %{
15912 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15913 __ string_compare($str1$$Register, $str2$$Register,
15914 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15915 $tmp1$$Register, $tmp2$$Register,
15916 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15917 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15918 StrIntrinsicNode::UU);
15919 %}
15920 ins_pipe(pipe_class_memory);
15921 %}
15922
15923 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15924 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15925 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15926 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15927 %{
15928 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15929 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15930 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15931 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15932 TEMP vtmp0, TEMP vtmp1, KILL cr);
15933 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15934 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15935
15936 ins_encode %{
15937 __ string_indexof($str1$$Register, $str2$$Register,
15938 $cnt1$$Register, $cnt2$$Register,
15939 $tmp1$$Register, $tmp2$$Register,
15940 $tmp3$$Register, $tmp4$$Register,
15941 $tmp5$$Register, $tmp6$$Register,
15942 -1, $result$$Register, StrIntrinsicNode::UU);
15943 %}
15944 ins_pipe(pipe_class_memory);
15945 %}
15946
15947 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15948 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15949 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15950 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15951 %{
15952 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15953 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15954 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15955 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15956 TEMP vtmp0, TEMP vtmp1, KILL cr);
15957 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15958 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15959
15960 ins_encode %{
15961 __ string_indexof($str1$$Register, $str2$$Register,
15962 $cnt1$$Register, $cnt2$$Register,
15963 $tmp1$$Register, $tmp2$$Register,
15964 $tmp3$$Register, $tmp4$$Register,
15965 $tmp5$$Register, $tmp6$$Register,
15966 -1, $result$$Register, StrIntrinsicNode::LL);
15967 %}
15968 ins_pipe(pipe_class_memory);
15969 %}
15970
15971 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15972 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15973 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15974 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15975 %{
15976 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15977 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15978 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15979 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15980 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15981 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15982 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15983
15984 ins_encode %{
15985 __ string_indexof($str1$$Register, $str2$$Register,
15986 $cnt1$$Register, $cnt2$$Register,
15987 $tmp1$$Register, $tmp2$$Register,
15988 $tmp3$$Register, $tmp4$$Register,
15989 $tmp5$$Register, $tmp6$$Register,
15990 -1, $result$$Register, StrIntrinsicNode::UL);
15991 %}
15992 ins_pipe(pipe_class_memory);
15993 %}
15994
15995 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15996 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15997 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15998 %{
15999 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16000 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16001 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16002 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16003 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16004 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16005
16006 ins_encode %{
16007 int icnt2 = (int)$int_cnt2$$constant;
16008 __ string_indexof($str1$$Register, $str2$$Register,
16009 $cnt1$$Register, zr,
16010 $tmp1$$Register, $tmp2$$Register,
16011 $tmp3$$Register, $tmp4$$Register, zr, zr,
16012 icnt2, $result$$Register, StrIntrinsicNode::UU);
16013 %}
16014 ins_pipe(pipe_class_memory);
16015 %}
16016
16017 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16018 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16019 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16020 %{
16021 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16022 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16023 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16024 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16025 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16026 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16027
16028 ins_encode %{
16029 int icnt2 = (int)$int_cnt2$$constant;
16030 __ string_indexof($str1$$Register, $str2$$Register,
16031 $cnt1$$Register, zr,
16032 $tmp1$$Register, $tmp2$$Register,
16033 $tmp3$$Register, $tmp4$$Register, zr, zr,
16034 icnt2, $result$$Register, StrIntrinsicNode::LL);
16035 %}
16036 ins_pipe(pipe_class_memory);
16037 %}
16038
16039 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16040 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16041 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16042 %{
16043 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16044 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16045 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16046 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16047 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16048 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16049
16050 ins_encode %{
16051 int icnt2 = (int)$int_cnt2$$constant;
16052 __ string_indexof($str1$$Register, $str2$$Register,
16053 $cnt1$$Register, zr,
16054 $tmp1$$Register, $tmp2$$Register,
16055 $tmp3$$Register, $tmp4$$Register, zr, zr,
16056 icnt2, $result$$Register, StrIntrinsicNode::UL);
16057 %}
16058 ins_pipe(pipe_class_memory);
16059 %}
16060
16061 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16062 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16063 iRegINoSp tmp3, rFlagsReg cr)
16064 %{
16065 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16066 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16067 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16068 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16069
16070 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16071
16072 ins_encode %{
16073 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16074 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16075 $tmp3$$Register);
16076 %}
16077 ins_pipe(pipe_class_memory);
16078 %}
16079
16080 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16081 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16082 iRegINoSp tmp3, rFlagsReg cr)
16083 %{
16084 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16085 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16086 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16087 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16088
16089 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16090
16091 ins_encode %{
16092 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16093 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16094 $tmp3$$Register);
16095 %}
16096 ins_pipe(pipe_class_memory);
16097 %}
16098
16099 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16100 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16101 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16102 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16103 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16104 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16105 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16106 ins_encode %{
16107 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16108 $result$$Register, $ztmp1$$FloatRegister,
16109 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16110 $ptmp$$PRegister, true /* isL */);
16111 %}
16112 ins_pipe(pipe_class_memory);
16113 %}
16114
16115 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16116 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16117 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16118 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16119 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16120 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16121 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16122 ins_encode %{
16123 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16124 $result$$Register, $ztmp1$$FloatRegister,
16125 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16126 $ptmp$$PRegister, false /* isL */);
16127 %}
16128 ins_pipe(pipe_class_memory);
16129 %}
16130
16131 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16132 iRegI_R0 result, rFlagsReg cr)
16133 %{
16134 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16135 match(Set result (StrEquals (Binary str1 str2) cnt));
16136 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16137
16138 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16139 ins_encode %{
16140 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16141 __ string_equals($str1$$Register, $str2$$Register,
16142 $result$$Register, $cnt$$Register);
16143 %}
16144 ins_pipe(pipe_class_memory);
16145 %}
16146
16147 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16148 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16149 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16150 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16151 iRegP_R10 tmp, rFlagsReg cr)
16152 %{
16153 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16154 match(Set result (AryEq ary1 ary2));
16155 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16156 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16157 TEMP vtmp6, TEMP vtmp7, KILL cr);
16158
16159 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16160 ins_encode %{
16161 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16162 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16163 $result$$Register, $tmp$$Register, 1);
16164 if (tpc == nullptr) {
16165 ciEnv::current()->record_failure("CodeCache is full");
16166 return;
16167 }
16168 %}
16169 ins_pipe(pipe_class_memory);
16170 %}
16171
16172 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16173 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16174 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16175 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16176 iRegP_R10 tmp, rFlagsReg cr)
16177 %{
16178 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16179 match(Set result (AryEq ary1 ary2));
16180 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16181 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16182 TEMP vtmp6, TEMP vtmp7, KILL cr);
16183
16184 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16185 ins_encode %{
16186 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16187 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16188 $result$$Register, $tmp$$Register, 2);
16189 if (tpc == nullptr) {
16190 ciEnv::current()->record_failure("CodeCache is full");
16191 return;
16192 }
16193 %}
16194 ins_pipe(pipe_class_memory);
16195 %}
16196
16197 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16198 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16199 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16200 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16201 %{
16202 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16203 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16204 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16205
16206 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16207 ins_encode %{
16208 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16209 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16210 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16211 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16212 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16213 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16214 (BasicType)$basic_type$$constant);
16215 if (tpc == nullptr) {
16216 ciEnv::current()->record_failure("CodeCache is full");
16217 return;
16218 }
16219 %}
16220 ins_pipe(pipe_class_memory);
16221 %}
16222
16223 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16224 %{
16225 match(Set result (CountPositives ary1 len));
16226 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16227 format %{ "count positives byte[] $ary1,$len -> $result" %}
16228 ins_encode %{
16229 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16230 if (tpc == nullptr) {
16231 ciEnv::current()->record_failure("CodeCache is full");
16232 return;
16233 }
16234 %}
16235 ins_pipe( pipe_slow );
16236 %}
16237
16238 // fast char[] to byte[] compression
16239 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16240 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16241 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16242 iRegI_R0 result, rFlagsReg cr)
16243 %{
16244 match(Set result (StrCompressedCopy src (Binary dst len)));
16245 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16246 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16247
16248 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16249 ins_encode %{
16250 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16251 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16252 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16253 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16254 %}
16255 ins_pipe(pipe_slow);
16256 %}
16257
16258 // fast byte[] to char[] inflation
16259 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16260 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16261 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16262 %{
16263 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16264 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16265 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16266 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16267
16268 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16269 ins_encode %{
16270 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16271 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16272 $vtmp2$$FloatRegister, $tmp$$Register);
16273 if (tpc == nullptr) {
16274 ciEnv::current()->record_failure("CodeCache is full");
16275 return;
16276 }
16277 %}
16278 ins_pipe(pipe_class_memory);
16279 %}
16280
16281 // encode char[] to byte[] in ISO_8859_1
16282 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16283 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16284 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16285 iRegI_R0 result, rFlagsReg cr)
16286 %{
16287 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16288 match(Set result (EncodeISOArray src (Binary dst len)));
16289 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16290 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16291
16292 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16293 ins_encode %{
16294 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16295 $result$$Register, false,
16296 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16297 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16298 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16299 %}
16300 ins_pipe(pipe_class_memory);
16301 %}
16302
16303 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16304 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16305 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16306 iRegI_R0 result, rFlagsReg cr)
16307 %{
16308 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16309 match(Set result (EncodeISOArray src (Binary dst len)));
16310 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16311 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16312
16313 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16314 ins_encode %{
16315 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16316 $result$$Register, true,
16317 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16318 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16319 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16320 %}
16321 ins_pipe(pipe_class_memory);
16322 %}
16323
16324 //----------------------------- CompressBits/ExpandBits ------------------------
16325
16326 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16327 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16328 match(Set dst (CompressBits src mask));
16329 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16330 format %{ "mov $tsrc, $src\n\t"
16331 "mov $tmask, $mask\n\t"
16332 "bext $tdst, $tsrc, $tmask\n\t"
16333 "mov $dst, $tdst"
16334 %}
16335 ins_encode %{
16336 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16337 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16338 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16339 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16340 %}
16341 ins_pipe(pipe_slow);
16342 %}
16343
16344 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16345 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16346 match(Set dst (CompressBits (LoadI mem) mask));
16347 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16348 format %{ "ldrs $tsrc, $mem\n\t"
16349 "ldrs $tmask, $mask\n\t"
16350 "bext $tdst, $tsrc, $tmask\n\t"
16351 "mov $dst, $tdst"
16352 %}
16353 ins_encode %{
16354 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16355 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16356 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16357 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16358 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16359 %}
16360 ins_pipe(pipe_slow);
16361 %}
16362
16363 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16364 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16365 match(Set dst (CompressBits src mask));
16366 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16367 format %{ "mov $tsrc, $src\n\t"
16368 "mov $tmask, $mask\n\t"
16369 "bext $tdst, $tsrc, $tmask\n\t"
16370 "mov $dst, $tdst"
16371 %}
16372 ins_encode %{
16373 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16374 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16375 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16376 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16377 %}
16378 ins_pipe(pipe_slow);
16379 %}
16380
16381 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16382 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16383 match(Set dst (CompressBits (LoadL mem) mask));
16384 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16385 format %{ "ldrd $tsrc, $mem\n\t"
16386 "ldrd $tmask, $mask\n\t"
16387 "bext $tdst, $tsrc, $tmask\n\t"
16388 "mov $dst, $tdst"
16389 %}
16390 ins_encode %{
16391 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16392 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16393 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16394 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16395 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16396 %}
16397 ins_pipe(pipe_slow);
16398 %}
16399
16400 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16401 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16402 match(Set dst (ExpandBits src mask));
16403 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16404 format %{ "mov $tsrc, $src\n\t"
16405 "mov $tmask, $mask\n\t"
16406 "bdep $tdst, $tsrc, $tmask\n\t"
16407 "mov $dst, $tdst"
16408 %}
16409 ins_encode %{
16410 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16411 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16412 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16413 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16414 %}
16415 ins_pipe(pipe_slow);
16416 %}
16417
16418 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16419 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16420 match(Set dst (ExpandBits (LoadI mem) mask));
16421 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16422 format %{ "ldrs $tsrc, $mem\n\t"
16423 "ldrs $tmask, $mask\n\t"
16424 "bdep $tdst, $tsrc, $tmask\n\t"
16425 "mov $dst, $tdst"
16426 %}
16427 ins_encode %{
16428 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16429 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16430 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16431 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16432 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16433 %}
16434 ins_pipe(pipe_slow);
16435 %}
16436
16437 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16438 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16439 match(Set dst (ExpandBits src mask));
16440 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16441 format %{ "mov $tsrc, $src\n\t"
16442 "mov $tmask, $mask\n\t"
16443 "bdep $tdst, $tsrc, $tmask\n\t"
16444 "mov $dst, $tdst"
16445 %}
16446 ins_encode %{
16447 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16448 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16449 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16450 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16451 %}
16452 ins_pipe(pipe_slow);
16453 %}
16454
16455
16456 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16457 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16458 match(Set dst (ExpandBits (LoadL mem) mask));
16459 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16460 format %{ "ldrd $tsrc, $mem\n\t"
16461 "ldrd $tmask, $mask\n\t"
16462 "bdep $tdst, $tsrc, $tmask\n\t"
16463 "mov $dst, $tdst"
16464 %}
16465 ins_encode %{
16466 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16467 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16468 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16469 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16470 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16471 %}
16472 ins_pipe(pipe_slow);
16473 %}
16474
16475 //----------------------------- Reinterpret ----------------------------------
16476 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16477 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16478 match(Set dst (ReinterpretHF2S src));
16479 format %{ "reinterpretHF2S $dst, $src" %}
16480 ins_encode %{
16481 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16482 %}
16483 ins_pipe(pipe_slow);
16484 %}
16485
16486 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16487 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16488 match(Set dst (ReinterpretS2HF src));
16489 format %{ "reinterpretS2HF $dst, $src" %}
16490 ins_encode %{
16491 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16492 %}
16493 ins_pipe(pipe_slow);
16494 %}
16495
16496 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16497 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16498 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16499 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16500 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16501 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16502 // can be omitted in this pattern, resulting in -
16503 // fcvt $dst, $src // Convert float to half-precision float
16504 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16505 %{
16506 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16507 format %{ "convF2HFAndS2HF $dst, $src" %}
16508 ins_encode %{
16509 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16510 %}
16511 ins_pipe(pipe_slow);
16512 %}
16513
16514 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16515 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16516 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16517 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16518 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16519 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16520 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16521 // resulting in -
16522 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16523 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16524 %{
16525 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16526 format %{ "convHF2SAndHF2F $dst, $src" %}
16527 ins_encode %{
16528 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16529 %}
16530 ins_pipe(pipe_slow);
16531 %}
16532
16533 // ============================================================================
16534 // This name is KNOWN by the ADLC and cannot be changed.
16535 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16536 // for this guy.
16537 instruct tlsLoadP(thread_RegP dst)
16538 %{
16539 match(Set dst (ThreadLocal));
16540
16541 ins_cost(0);
16542
16543 format %{ " -- \t// $dst=Thread::current(), empty" %}
16544
16545 size(0);
16546
16547 ins_encode( /*empty*/ );
16548
16549 ins_pipe(pipe_class_empty);
16550 %}
16551
16552 //----------PEEPHOLE RULES-----------------------------------------------------
16553 // These must follow all instruction definitions as they use the names
16554 // defined in the instructions definitions.
16555 //
16556 // peepmatch ( root_instr_name [preceding_instruction]* );
16557 //
16558 // peepconstraint %{
16559 // (instruction_number.operand_name relational_op instruction_number.operand_name
16560 // [, ...] );
16561 // // instruction numbers are zero-based using left to right order in peepmatch
16562 //
16563 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16564 // // provide an instruction_number.operand_name for each operand that appears
16565 // // in the replacement instruction's match rule
16566 //
16567 // ---------VM FLAGS---------------------------------------------------------
16568 //
16569 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16570 //
16571 // Each peephole rule is given an identifying number starting with zero and
16572 // increasing by one in the order seen by the parser. An individual peephole
16573 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16574 // on the command-line.
16575 //
16576 // ---------CURRENT LIMITATIONS----------------------------------------------
16577 //
16578 // Only match adjacent instructions in same basic block
16579 // Only equality constraints
16580 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16581 // Only one replacement instruction
16582 //
16583 // ---------EXAMPLE----------------------------------------------------------
16584 //
16585 // // pertinent parts of existing instructions in architecture description
16586 // instruct movI(iRegINoSp dst, iRegI src)
16587 // %{
16588 // match(Set dst (CopyI src));
16589 // %}
16590 //
16591 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16592 // %{
16593 // match(Set dst (AddI dst src));
16594 // effect(KILL cr);
16595 // %}
16596 //
16597 // // Change (inc mov) to lea
16598 // peephole %{
16599 // // increment preceded by register-register move
16600 // peepmatch ( incI_iReg movI );
16601 // // require that the destination register of the increment
16602 // // match the destination register of the move
16603 // peepconstraint ( 0.dst == 1.dst );
16604 // // construct a replacement instruction that sets
16605 // // the destination to ( move's source register + one )
16606 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16607 // %}
16608 //
16609
16610 // Implementation no longer uses movX instructions since
16611 // machine-independent system no longer uses CopyX nodes.
16612 //
16613 // peephole
16614 // %{
16615 // peepmatch (incI_iReg movI);
16616 // peepconstraint (0.dst == 1.dst);
16617 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16618 // %}
16619
16620 // peephole
16621 // %{
16622 // peepmatch (decI_iReg movI);
16623 // peepconstraint (0.dst == 1.dst);
16624 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16625 // %}
16626
16627 // peephole
16628 // %{
16629 // peepmatch (addI_iReg_imm movI);
16630 // peepconstraint (0.dst == 1.dst);
16631 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16632 // %}
16633
16634 // peephole
16635 // %{
16636 // peepmatch (incL_iReg movL);
16637 // peepconstraint (0.dst == 1.dst);
16638 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16639 // %}
16640
16641 // peephole
16642 // %{
16643 // peepmatch (decL_iReg movL);
16644 // peepconstraint (0.dst == 1.dst);
16645 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16646 // %}
16647
16648 // peephole
16649 // %{
16650 // peepmatch (addL_iReg_imm movL);
16651 // peepconstraint (0.dst == 1.dst);
16652 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16653 // %}
16654
16655 // peephole
16656 // %{
16657 // peepmatch (addP_iReg_imm movP);
16658 // peepconstraint (0.dst == 1.dst);
16659 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16660 // %}
16661
16662 // // Change load of spilled value to only a spill
16663 // instruct storeI(memory mem, iRegI src)
16664 // %{
16665 // match(Set mem (StoreI mem src));
16666 // %}
16667 //
16668 // instruct loadI(iRegINoSp dst, memory mem)
16669 // %{
16670 // match(Set dst (LoadI mem));
16671 // %}
16672 //
16673
16674 //----------SMARTSPILL RULES---------------------------------------------------
16675 // These must follow all instruction definitions as they use the names
16676 // defined in the instructions definitions.
16677
16678 // Local Variables:
16679 // mode: c++
16680 // End: