1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else if (_entry_point == nullptr) {
1694 // See CallLeafNoFPIndirect
1695 return 1 * NativeInstruction::instruction_size;
1696 } else {
1697 return 6 * NativeInstruction::instruction_size;
1698 }
1699 }
1700
1701 //=============================================================================
1702
1703 #ifndef PRODUCT
1704 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1705 st->print("BREAKPOINT");
1706 }
1707 #endif
1708
1709 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1710 __ brk(0);
1711 }
1712
1713 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1714 return MachNode::size(ra_);
1715 }
1716
1717 //=============================================================================
1718
1719 #ifndef PRODUCT
1720 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1721 st->print("nop \t# %d bytes pad for loops and calls", _count);
1722 }
1723 #endif
1724
1725 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1726 for (int i = 0; i < _count; i++) {
1727 __ nop();
1728 }
1729 }
1730
1731 uint MachNopNode::size(PhaseRegAlloc*) const {
1732 return _count * NativeInstruction::instruction_size;
1733 }
1734
1735 //=============================================================================
1736 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1737
1738 int ConstantTable::calculate_table_base_offset() const {
1739 return 0; // absolute addressing, no offset
1740 }
1741
1742 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1743 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1744 ShouldNotReachHere();
1745 }
1746
1747 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1748 // Empty encoding
1749 }
1750
1751 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1752 return 0;
1753 }
1754
1755 #ifndef PRODUCT
1756 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1757 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1758 }
1759 #endif
1760
1761 #ifndef PRODUCT
1762 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1763 Compile* C = ra_->C;
1764
1765 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1766
1767 if (C->output()->need_stack_bang(framesize))
1768 st->print("# stack bang size=%d\n\t", framesize);
1769
1770 if (VM_Version::use_rop_protection()) {
1771 st->print("ldr zr, [lr]\n\t");
1772 st->print("paciaz\n\t");
1773 }
1774 if (framesize < ((1 << 9) + 2 * wordSize)) {
1775 st->print("sub sp, sp, #%d\n\t", framesize);
1776 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1777 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1778 } else {
1779 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1780 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1781 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1782 st->print("sub sp, sp, rscratch1");
1783 }
1784 if (C->stub_function() == nullptr) {
1785 st->print("\n\t");
1786 st->print("ldr rscratch1, [guard]\n\t");
1787 st->print("dmb ishld\n\t");
1788 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1789 st->print("cmp rscratch1, rscratch2\n\t");
1790 st->print("b.eq skip");
1791 st->print("\n\t");
1792 st->print("blr #nmethod_entry_barrier_stub\n\t");
1793 st->print("b skip\n\t");
1794 st->print("guard: int\n\t");
1795 st->print("\n\t");
1796 st->print("skip:\n\t");
1797 }
1798 }
1799 #endif
1800
1801 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1802 Compile* C = ra_->C;
1803
1804
1805 __ verified_entry(C, 0);
1806
1807 if (C->stub_function() == nullptr) {
1808 __ entry_barrier();
1809 }
1810
1811 if (!Compile::current()->output()->in_scratch_emit_size()) {
1812 __ bind(*_verified_entry);
1813 }
1814
1815 if (VerifyStackAtCalls) {
1816 Unimplemented();
1817 }
1818
1819 C->output()->set_frame_complete(__ offset());
1820
1821 if (C->has_mach_constant_base_node()) {
1822 // NOTE: We set the table base offset here because users might be
1823 // emitted before MachConstantBaseNode.
1824 ConstantTable& constant_table = C->output()->constant_table();
1825 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1826 }
1827 }
1828
1829 int MachPrologNode::reloc() const
1830 {
1831 return 0;
1832 }
1833
1834 //=============================================================================
1835
1836 #ifndef PRODUCT
1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1838 Compile* C = ra_->C;
1839 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1840
1841 st->print("# pop frame %d\n\t",framesize);
1842
1843 if (framesize == 0) {
1844 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1845 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1846 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1847 st->print("add sp, sp, #%d\n\t", framesize);
1848 } else {
1849 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1850 st->print("add sp, sp, rscratch1\n\t");
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 }
1853 if (VM_Version::use_rop_protection()) {
1854 st->print("autiaz\n\t");
1855 st->print("ldr zr, [lr]\n\t");
1856 }
1857
1858 if (do_polling() && C->is_method_compilation()) {
1859 st->print("# test polling word\n\t");
1860 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1861 st->print("cmp sp, rscratch1\n\t");
1862 st->print("bhi #slow_path");
1863 }
1864 }
1865 #endif
1866
1867 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1868 Compile* C = ra_->C;
1869 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1870
1871 __ remove_frame(framesize, C->needs_stack_repair());
1872
1873 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1874 __ reserved_stack_check();
1875 }
1876
1877 if (do_polling() && C->is_method_compilation()) {
1878 Label dummy_label;
1879 Label* code_stub = &dummy_label;
1880 if (!C->output()->in_scratch_emit_size()) {
1881 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1882 C->output()->add_stub(stub);
1883 code_stub = &stub->entry();
1884 }
1885 __ relocate(relocInfo::poll_return_type);
1886 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1887 }
1888 }
1889
1890 int MachEpilogNode::reloc() const {
1891 // Return number of relocatable values contained in this instruction.
1892 return 1; // 1 for polling page.
1893 }
1894
1895 const Pipeline * MachEpilogNode::pipeline() const {
1896 return MachNode::pipeline_class();
1897 }
1898
1899 //=============================================================================
1900
1901 static enum RC rc_class(OptoReg::Name reg) {
1902
1903 if (reg == OptoReg::Bad) {
1904 return rc_bad;
1905 }
1906
1907 // we have 32 int registers * 2 halves
1908 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1909
1910 if (reg < slots_of_int_registers) {
1911 return rc_int;
1912 }
1913
1914 // we have 32 float register * 8 halves
1915 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1916 if (reg < slots_of_int_registers + slots_of_float_registers) {
1917 return rc_float;
1918 }
1919
1920 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1921 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1922 return rc_predicate;
1923 }
1924
1925 // Between predicate regs & stack is the flags.
1926 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1927
1928 return rc_stack;
1929 }
1930
1931 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1932 Compile* C = ra_->C;
1933
1934 // Get registers to move.
1935 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1936 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1937 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1938 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1939
1940 enum RC src_hi_rc = rc_class(src_hi);
1941 enum RC src_lo_rc = rc_class(src_lo);
1942 enum RC dst_hi_rc = rc_class(dst_hi);
1943 enum RC dst_lo_rc = rc_class(dst_lo);
1944
1945 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1946
1947 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1948 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1949 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1950 "expected aligned-adjacent pairs");
1951 }
1952
1953 if (src_lo == dst_lo && src_hi == dst_hi) {
1954 return 0; // Self copy, no move.
1955 }
1956
1957 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1958 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1959 int src_offset = ra_->reg2offset(src_lo);
1960 int dst_offset = ra_->reg2offset(dst_lo);
1961
1962 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1963 uint ireg = ideal_reg();
1964 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
1965 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
1966 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
1967 if (ireg == Op_VecA && masm) {
1968 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1969 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1970 // stack->stack
1971 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1972 sve_vector_reg_size_in_bytes);
1973 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1974 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1975 sve_vector_reg_size_in_bytes);
1976 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1977 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1978 sve_vector_reg_size_in_bytes);
1979 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1980 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1981 as_FloatRegister(Matcher::_regEncode[src_lo]),
1982 as_FloatRegister(Matcher::_regEncode[src_lo]));
1983 } else {
1984 ShouldNotReachHere();
1985 }
1986 } else if (masm) {
1987 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1988 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1989 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1990 // stack->stack
1991 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1992 if (ireg == Op_VecD) {
1993 __ unspill(rscratch1, true, src_offset);
1994 __ spill(rscratch1, true, dst_offset);
1995 } else {
1996 __ spill_copy128(src_offset, dst_offset);
1997 }
1998 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1999 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2000 ireg == Op_VecD ? __ T8B : __ T16B,
2001 as_FloatRegister(Matcher::_regEncode[src_lo]));
2002 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2003 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2004 ireg == Op_VecD ? __ D : __ Q,
2005 ra_->reg2offset(dst_lo));
2006 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2007 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2008 ireg == Op_VecD ? __ D : __ Q,
2009 ra_->reg2offset(src_lo));
2010 } else {
2011 ShouldNotReachHere();
2012 }
2013 }
2014 } else if (masm) {
2015 switch (src_lo_rc) {
2016 case rc_int:
2017 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2018 if (is64) {
2019 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2020 as_Register(Matcher::_regEncode[src_lo]));
2021 } else {
2022 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2023 as_Register(Matcher::_regEncode[src_lo]));
2024 }
2025 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2026 if (is64) {
2027 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2028 as_Register(Matcher::_regEncode[src_lo]));
2029 } else {
2030 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2031 as_Register(Matcher::_regEncode[src_lo]));
2032 }
2033 } else { // gpr --> stack spill
2034 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2035 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2036 }
2037 break;
2038 case rc_float:
2039 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2040 if (is64) {
2041 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2042 as_FloatRegister(Matcher::_regEncode[src_lo]));
2043 } else {
2044 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2045 as_FloatRegister(Matcher::_regEncode[src_lo]));
2046 }
2047 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2048 if (is64) {
2049 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 as_FloatRegister(Matcher::_regEncode[src_lo]));
2051 } else {
2052 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2053 as_FloatRegister(Matcher::_regEncode[src_lo]));
2054 }
2055 } else { // fpr --> stack spill
2056 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2057 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2058 is64 ? __ D : __ S, dst_offset);
2059 }
2060 break;
2061 case rc_stack:
2062 if (dst_lo_rc == rc_int) { // stack --> gpr load
2063 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2064 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2065 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2066 is64 ? __ D : __ S, src_offset);
2067 } else if (dst_lo_rc == rc_predicate) {
2068 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2069 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2070 } else { // stack --> stack copy
2071 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2072 if (ideal_reg() == Op_RegVectMask) {
2073 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2074 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2075 } else {
2076 __ unspill(rscratch1, is64, src_offset);
2077 __ spill(rscratch1, is64, dst_offset);
2078 }
2079 }
2080 break;
2081 case rc_predicate:
2082 if (dst_lo_rc == rc_predicate) {
2083 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2084 } else if (dst_lo_rc == rc_stack) {
2085 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2086 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2087 } else {
2088 assert(false, "bad src and dst rc_class combination.");
2089 ShouldNotReachHere();
2090 }
2091 break;
2092 default:
2093 assert(false, "bad rc_class for spill");
2094 ShouldNotReachHere();
2095 }
2096 }
2097
2098 if (st) {
2099 st->print("spill ");
2100 if (src_lo_rc == rc_stack) {
2101 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2102 } else {
2103 st->print("%s -> ", Matcher::regName[src_lo]);
2104 }
2105 if (dst_lo_rc == rc_stack) {
2106 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2107 } else {
2108 st->print("%s", Matcher::regName[dst_lo]);
2109 }
2110 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2111 int vsize = 0;
2112 switch (ideal_reg()) {
2113 case Op_VecD:
2114 vsize = 64;
2115 break;
2116 case Op_VecX:
2117 vsize = 128;
2118 break;
2119 case Op_VecA:
2120 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2121 break;
2122 default:
2123 assert(false, "bad register type for spill");
2124 ShouldNotReachHere();
2125 }
2126 st->print("\t# vector spill size = %d", vsize);
2127 } else if (ideal_reg() == Op_RegVectMask) {
2128 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2129 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2130 st->print("\t# predicate spill size = %d", vsize);
2131 } else {
2132 st->print("\t# spill size = %d", is64 ? 64 : 32);
2133 }
2134 }
2135
2136 return 0;
2137
2138 }
2139
2140 #ifndef PRODUCT
2141 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2142 if (!ra_)
2143 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2144 else
2145 implementation(nullptr, ra_, false, st);
2146 }
2147 #endif
2148
2149 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2150 implementation(masm, ra_, false, nullptr);
2151 }
2152
2153 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2154 return MachNode::size(ra_);
2155 }
2156
2157 //=============================================================================
2158
2159 #ifndef PRODUCT
2160 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2161 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2162 int reg = ra_->get_reg_first(this);
2163 st->print("add %s, rsp, #%d]\t# box lock",
2164 Matcher::regName[reg], offset);
2165 }
2166 #endif
2167
2168 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2169 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2170 int reg = ra_->get_encode(this);
2171
2172 // This add will handle any 24-bit signed offset. 24 bits allows an
2173 // 8 megabyte stack frame.
2174 __ add(as_Register(reg), sp, offset);
2175 }
2176
2177 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2178 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2179 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2180
2181 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2182 return NativeInstruction::instruction_size;
2183 } else {
2184 return 2 * NativeInstruction::instruction_size;
2185 }
2186 }
2187
2188 ///=============================================================================
2189 #ifndef PRODUCT
2190 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2191 {
2192 st->print_cr("# MachVEPNode");
2193 if (!_verified) {
2194 st->print_cr("\t load_class");
2195 } else {
2196 st->print_cr("\t unpack_inline_arg");
2197 }
2198 }
2199 #endif
2200
2201 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2202 {
2203 if (!_verified) {
2204 __ ic_check(1);
2205 } else {
2206 // TODO 8284443 Avoid creation of temporary frame
2207 if (ra_->C->stub_function() == nullptr) {
2208 __ verified_entry(ra_->C, 0);
2209 __ entry_barrier();
2210 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2211 __ remove_frame(framesize, false);
2212 }
2213 // Unpack inline type args passed as oop and then jump to
2214 // the verified entry point (skipping the unverified entry).
2215 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2216 // Emit code for verified entry and save increment for stack repair on return
2217 __ verified_entry(ra_->C, sp_inc);
2218 if (Compile::current()->output()->in_scratch_emit_size()) {
2219 Label dummy_verified_entry;
2220 __ b(dummy_verified_entry);
2221 } else {
2222 __ b(*_verified_entry);
2223 }
2224 }
2225 }
2226
2227 //=============================================================================
2228 #ifndef PRODUCT
2229 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2230 {
2231 st->print_cr("# MachUEPNode");
2232 if (UseCompressedClassPointers) {
2233 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2234 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2235 st->print_cr("\tcmpw rscratch1, r10");
2236 } else {
2237 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2238 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2239 st->print_cr("\tcmp rscratch1, r10");
2240 }
2241 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2242 }
2243 #endif
2244
2245 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2246 {
2247 __ ic_check(InteriorEntryAlignment);
2248 }
2249
2250 // REQUIRED EMIT CODE
2251
2252 //=============================================================================
2253
2254 // Emit deopt handler code.
2255 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2256 {
2257 // Note that the code buffer's insts_mark is always relative to insts.
2258 // That's why we must use the macroassembler to generate a handler.
2259 address base = __ start_a_stub(size_deopt_handler());
2260 if (base == nullptr) {
2261 ciEnv::current()->record_failure("CodeCache is full");
2262 return 0; // CodeBuffer::expand failed
2263 }
2264
2265 int offset = __ offset();
2266 Label start;
2267 __ bind(start);
2268 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2269
2270 int entry_offset = __ offset();
2271 __ b(start);
2272
2273 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2274 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2275 "out of bounds read in post-call NOP check");
2276 __ end_a_stub();
2277 return entry_offset;
2278 }
2279
2280 // REQUIRED MATCHER CODE
2281
2282 //=============================================================================
2283
2284 bool Matcher::match_rule_supported(int opcode) {
2285 if (!has_match_rule(opcode))
2286 return false;
2287
2288 switch (opcode) {
2289 case Op_OnSpinWait:
2290 return VM_Version::supports_on_spin_wait();
2291 case Op_CacheWB:
2292 case Op_CacheWBPreSync:
2293 case Op_CacheWBPostSync:
2294 if (!VM_Version::supports_data_cache_line_flush()) {
2295 return false;
2296 }
2297 break;
2298 case Op_ExpandBits:
2299 case Op_CompressBits:
2300 if (!VM_Version::supports_svebitperm()) {
2301 return false;
2302 }
2303 break;
2304 case Op_FmaF:
2305 case Op_FmaD:
2306 case Op_FmaVF:
2307 case Op_FmaVD:
2308 if (!UseFMA) {
2309 return false;
2310 }
2311 break;
2312 case Op_FmaHF:
2313 // UseFMA flag also needs to be checked along with FEAT_FP16
2314 if (!UseFMA || !is_feat_fp16_supported()) {
2315 return false;
2316 }
2317 break;
2318 case Op_AddHF:
2319 case Op_SubHF:
2320 case Op_MulHF:
2321 case Op_DivHF:
2322 case Op_MinHF:
2323 case Op_MaxHF:
2324 case Op_SqrtHF:
2325 // Half-precision floating point scalar operations require FEAT_FP16
2326 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2327 // features are supported.
2328 if (!is_feat_fp16_supported()) {
2329 return false;
2330 }
2331 break;
2332 }
2333
2334 return true; // Per default match rules are supported.
2335 }
2336
2337 const RegMask* Matcher::predicate_reg_mask(void) {
2338 return &_PR_REG_mask;
2339 }
2340
2341 bool Matcher::supports_vector_calling_convention(void) {
2342 return EnableVectorSupport;
2343 }
2344
2345 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2346 assert(EnableVectorSupport, "sanity");
2347 int lo = V0_num;
2348 int hi = V0_H_num;
2349 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2350 hi = V0_K_num;
2351 }
2352 return OptoRegPair(hi, lo);
2353 }
2354
2355 // Is this branch offset short enough that a short branch can be used?
2356 //
2357 // NOTE: If the platform does not provide any short branch variants, then
2358 // this method should return false for offset 0.
2359 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2360 // The passed offset is relative to address of the branch.
2361
2362 return (-32768 <= offset && offset < 32768);
2363 }
2364
2365 // Vector width in bytes.
2366 int Matcher::vector_width_in_bytes(BasicType bt) {
2367 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2368 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2369 // Minimum 2 values in vector
2370 if (size < 2*type2aelembytes(bt)) size = 0;
2371 // But never < 4
2372 if (size < 4) size = 0;
2373 return size;
2374 }
2375
2376 // Limits on vector size (number of elements) loaded into vector.
2377 int Matcher::max_vector_size(const BasicType bt) {
2378 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2379 }
2380
2381 int Matcher::min_vector_size(const BasicType bt) {
2382 // Usually, the shortest vector length supported by AArch64 ISA and
2383 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2384 // vectors in a few special cases.
2385 int size;
2386 switch(bt) {
2387 case T_BOOLEAN:
2388 // Load/store a vector mask with only 2 elements for vector types
2389 // such as "2I/2F/2L/2D".
2390 size = 2;
2391 break;
2392 case T_BYTE:
2393 // Generate a "4B" vector, to support vector cast between "8B/16B"
2394 // and "4S/4I/4L/4F/4D".
2395 size = 4;
2396 break;
2397 case T_SHORT:
2398 // Generate a "2S" vector, to support vector cast between "4S/8S"
2399 // and "2I/2L/2F/2D".
2400 size = 2;
2401 break;
2402 default:
2403 // Limit the min vector length to 64-bit.
2404 size = 8 / type2aelembytes(bt);
2405 // The number of elements in a vector should be at least 2.
2406 size = MAX2(size, 2);
2407 }
2408
2409 int max_size = max_vector_size(bt);
2410 return MIN2(size, max_size);
2411 }
2412
2413 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2414 return Matcher::max_vector_size(bt);
2415 }
2416
2417 // Actual max scalable vector register length.
2418 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2419 return Matcher::max_vector_size(bt);
2420 }
2421
2422 // Vector ideal reg.
2423 uint Matcher::vector_ideal_reg(int len) {
2424 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2425 return Op_VecA;
2426 }
2427 switch(len) {
2428 // For 16-bit/32-bit mask vector, reuse VecD.
2429 case 2:
2430 case 4:
2431 case 8: return Op_VecD;
2432 case 16: return Op_VecX;
2433 }
2434 ShouldNotReachHere();
2435 return 0;
2436 }
2437
2438 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2439 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2440 switch (ideal_reg) {
2441 case Op_VecA: return new vecAOper();
2442 case Op_VecD: return new vecDOper();
2443 case Op_VecX: return new vecXOper();
2444 }
2445 ShouldNotReachHere();
2446 return nullptr;
2447 }
2448
2449 bool Matcher::is_reg2reg_move(MachNode* m) {
2450 return false;
2451 }
2452
2453 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2454 return false;
2455 }
2456
2457 bool Matcher::is_generic_vector(MachOper* opnd) {
2458 return opnd->opcode() == VREG;
2459 }
2460
2461 #ifdef ASSERT
2462 // Return whether or not this register is ever used as an argument.
2463 bool Matcher::can_be_java_arg(int reg)
2464 {
2465 return
2466 reg == R0_num || reg == R0_H_num ||
2467 reg == R1_num || reg == R1_H_num ||
2468 reg == R2_num || reg == R2_H_num ||
2469 reg == R3_num || reg == R3_H_num ||
2470 reg == R4_num || reg == R4_H_num ||
2471 reg == R5_num || reg == R5_H_num ||
2472 reg == R6_num || reg == R6_H_num ||
2473 reg == R7_num || reg == R7_H_num ||
2474 reg == V0_num || reg == V0_H_num ||
2475 reg == V1_num || reg == V1_H_num ||
2476 reg == V2_num || reg == V2_H_num ||
2477 reg == V3_num || reg == V3_H_num ||
2478 reg == V4_num || reg == V4_H_num ||
2479 reg == V5_num || reg == V5_H_num ||
2480 reg == V6_num || reg == V6_H_num ||
2481 reg == V7_num || reg == V7_H_num;
2482 }
2483 #endif
2484
2485 uint Matcher::int_pressure_limit()
2486 {
2487 // JDK-8183543: When taking the number of available registers as int
2488 // register pressure threshold, the jtreg test:
2489 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2490 // failed due to C2 compilation failure with
2491 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2492 //
2493 // A derived pointer is live at CallNode and then is flagged by RA
2494 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2495 // derived pointers and lastly fail to spill after reaching maximum
2496 // number of iterations. Lowering the default pressure threshold to
2497 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2498 // a high register pressure area of the code so that split_DEF can
2499 // generate DefinitionSpillCopy for the derived pointer.
2500 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2501 if (!PreserveFramePointer) {
2502 // When PreserveFramePointer is off, frame pointer is allocatable,
2503 // but different from other SOC registers, it is excluded from
2504 // fatproj's mask because its save type is No-Save. Decrease 1 to
2505 // ensure high pressure at fatproj when PreserveFramePointer is off.
2506 // See check_pressure_at_fatproj().
2507 default_int_pressure_threshold--;
2508 }
2509 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2510 }
2511
2512 uint Matcher::float_pressure_limit()
2513 {
2514 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2515 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2516 }
2517
2518 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2519 return false;
2520 }
2521
2522 const RegMask& Matcher::divI_proj_mask() {
2523 ShouldNotReachHere();
2524 return RegMask::EMPTY;
2525 }
2526
2527 // Register for MODI projection of divmodI.
2528 const RegMask& Matcher::modI_proj_mask() {
2529 ShouldNotReachHere();
2530 return RegMask::EMPTY;
2531 }
2532
2533 // Register for DIVL projection of divmodL.
2534 const RegMask& Matcher::divL_proj_mask() {
2535 ShouldNotReachHere();
2536 return RegMask::EMPTY;
2537 }
2538
2539 // Register for MODL projection of divmodL.
2540 const RegMask& Matcher::modL_proj_mask() {
2541 ShouldNotReachHere();
2542 return RegMask::EMPTY;
2543 }
2544
2545 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2546 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2547 Node* u = addp->fast_out(i);
2548 if (u->is_LoadStore()) {
2549 // On AArch64, LoadStoreNodes (i.e. compare and swap
2550 // instructions) only take register indirect as an operand, so
2551 // any attempt to use an AddPNode as an input to a LoadStoreNode
2552 // must fail.
2553 return false;
2554 }
2555 if (u->is_Mem()) {
2556 int opsize = u->as_Mem()->memory_size();
2557 assert(opsize > 0, "unexpected memory operand size");
2558 if (u->as_Mem()->memory_size() != (1<<shift)) {
2559 return false;
2560 }
2561 }
2562 }
2563 return true;
2564 }
2565
2566 // Convert BoolTest condition to Assembler condition.
2567 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2568 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2569 Assembler::Condition result;
2570 switch(cond) {
2571 case BoolTest::eq:
2572 result = Assembler::EQ; break;
2573 case BoolTest::ne:
2574 result = Assembler::NE; break;
2575 case BoolTest::le:
2576 result = Assembler::LE; break;
2577 case BoolTest::ge:
2578 result = Assembler::GE; break;
2579 case BoolTest::lt:
2580 result = Assembler::LT; break;
2581 case BoolTest::gt:
2582 result = Assembler::GT; break;
2583 case BoolTest::ule:
2584 result = Assembler::LS; break;
2585 case BoolTest::uge:
2586 result = Assembler::HS; break;
2587 case BoolTest::ult:
2588 result = Assembler::LO; break;
2589 case BoolTest::ugt:
2590 result = Assembler::HI; break;
2591 case BoolTest::overflow:
2592 result = Assembler::VS; break;
2593 case BoolTest::no_overflow:
2594 result = Assembler::VC; break;
2595 default:
2596 ShouldNotReachHere();
2597 return Assembler::Condition(-1);
2598 }
2599
2600 // Check conversion
2601 if (cond & BoolTest::unsigned_compare) {
2602 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2603 } else {
2604 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2605 }
2606
2607 return result;
2608 }
2609
2610 // Binary src (Replicate con)
2611 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2612 if (n == nullptr || m == nullptr) {
2613 return false;
2614 }
2615
2616 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2617 return false;
2618 }
2619
2620 Node* imm_node = m->in(1);
2621 if (!imm_node->is_Con()) {
2622 return false;
2623 }
2624
2625 const Type* t = imm_node->bottom_type();
2626 if (!(t->isa_int() || t->isa_long())) {
2627 return false;
2628 }
2629
2630 switch (n->Opcode()) {
2631 case Op_AndV:
2632 case Op_OrV:
2633 case Op_XorV: {
2634 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2635 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2636 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2637 }
2638 case Op_AddVB:
2639 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2640 case Op_AddVS:
2641 case Op_AddVI:
2642 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2643 case Op_AddVL:
2644 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2645 default:
2646 return false;
2647 }
2648 }
2649
2650 // (XorV src (Replicate m1))
2651 // (XorVMask src (MaskAll m1))
2652 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2653 if (n != nullptr && m != nullptr) {
2654 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2655 VectorNode::is_all_ones_vector(m);
2656 }
2657 return false;
2658 }
2659
2660 // Should the matcher clone input 'm' of node 'n'?
2661 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2662 if (is_vshift_con_pattern(n, m) ||
2663 is_vector_bitwise_not_pattern(n, m) ||
2664 is_valid_sve_arith_imm_pattern(n, m) ||
2665 is_encode_and_store_pattern(n, m)) {
2666 mstack.push(m, Visit);
2667 return true;
2668 }
2669 return false;
2670 }
2671
2672 // Should the Matcher clone shifts on addressing modes, expecting them
2673 // to be subsumed into complex addressing expressions or compute them
2674 // into registers?
2675 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2676
2677 // Loads and stores with indirect memory input (e.g., volatile loads and
2678 // stores) do not subsume the input into complex addressing expressions. If
2679 // the addressing expression is input to at least one such load or store, do
2680 // not clone the addressing expression. Query needs_acquiring_load and
2681 // needs_releasing_store as a proxy for indirect memory input, as it is not
2682 // possible to directly query for indirect memory input at this stage.
2683 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2684 Node* n = m->fast_out(i);
2685 if (n->is_Load() && needs_acquiring_load(n)) {
2686 return false;
2687 }
2688 if (n->is_Store() && needs_releasing_store(n)) {
2689 return false;
2690 }
2691 }
2692
2693 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2694 return true;
2695 }
2696
2697 Node *off = m->in(AddPNode::Offset);
2698 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2699 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2700 // Are there other uses besides address expressions?
2701 !is_visited(off)) {
2702 address_visited.set(off->_idx); // Flag as address_visited
2703 mstack.push(off->in(2), Visit);
2704 Node *conv = off->in(1);
2705 if (conv->Opcode() == Op_ConvI2L &&
2706 // Are there other uses besides address expressions?
2707 !is_visited(conv)) {
2708 address_visited.set(conv->_idx); // Flag as address_visited
2709 mstack.push(conv->in(1), Pre_Visit);
2710 } else {
2711 mstack.push(conv, Pre_Visit);
2712 }
2713 address_visited.test_set(m->_idx); // Flag as address_visited
2714 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2715 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2716 return true;
2717 } else if (off->Opcode() == Op_ConvI2L &&
2718 // Are there other uses besides address expressions?
2719 !is_visited(off)) {
2720 address_visited.test_set(m->_idx); // Flag as address_visited
2721 address_visited.set(off->_idx); // Flag as address_visited
2722 mstack.push(off->in(1), Pre_Visit);
2723 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2724 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2725 return true;
2726 }
2727 return false;
2728 }
2729
2730 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2731 { \
2732 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2733 guarantee(DISP == 0, "mode not permitted for volatile"); \
2734 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2735 __ INSN(REG, as_Register(BASE)); \
2736 }
2737
2738
2739 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2740 {
2741 Address::extend scale;
2742
2743 // Hooboy, this is fugly. We need a way to communicate to the
2744 // encoder that the index needs to be sign extended, so we have to
2745 // enumerate all the cases.
2746 switch (opcode) {
2747 case INDINDEXSCALEDI2L:
2748 case INDINDEXSCALEDI2LN:
2749 case INDINDEXI2L:
2750 case INDINDEXI2LN:
2751 scale = Address::sxtw(size);
2752 break;
2753 default:
2754 scale = Address::lsl(size);
2755 }
2756
2757 if (index == -1) {
2758 return Address(base, disp);
2759 } else {
2760 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2761 return Address(base, as_Register(index), scale);
2762 }
2763 }
2764
2765
2766 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2767 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2768 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2769 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2770 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2771
2772 // Used for all non-volatile memory accesses. The use of
2773 // $mem->opcode() to discover whether this pattern uses sign-extended
2774 // offsets is something of a kludge.
2775 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2776 Register reg, int opcode,
2777 Register base, int index, int scale, int disp,
2778 int size_in_memory)
2779 {
2780 Address addr = mem2address(opcode, base, index, scale, disp);
2781 if (addr.getMode() == Address::base_plus_offset) {
2782 /* Fix up any out-of-range offsets. */
2783 assert_different_registers(rscratch1, base);
2784 assert_different_registers(rscratch1, reg);
2785 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2786 }
2787 (masm->*insn)(reg, addr);
2788 }
2789
2790 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2791 FloatRegister reg, int opcode,
2792 Register base, int index, int size, int disp,
2793 int size_in_memory)
2794 {
2795 Address::extend scale;
2796
2797 switch (opcode) {
2798 case INDINDEXSCALEDI2L:
2799 case INDINDEXSCALEDI2LN:
2800 scale = Address::sxtw(size);
2801 break;
2802 default:
2803 scale = Address::lsl(size);
2804 }
2805
2806 if (index == -1) {
2807 // Fix up any out-of-range offsets.
2808 assert_different_registers(rscratch1, base);
2809 Address addr = Address(base, disp);
2810 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2811 (masm->*insn)(reg, addr);
2812 } else {
2813 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2814 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2815 }
2816 }
2817
2818 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2819 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2820 int opcode, Register base, int index, int size, int disp)
2821 {
2822 if (index == -1) {
2823 (masm->*insn)(reg, T, Address(base, disp));
2824 } else {
2825 assert(disp == 0, "unsupported address mode");
2826 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2827 }
2828 }
2829
2830 %}
2831
2832
2833
2834 //----------ENCODING BLOCK-----------------------------------------------------
2835 // This block specifies the encoding classes used by the compiler to
2836 // output byte streams. Encoding classes are parameterized macros
2837 // used by Machine Instruction Nodes in order to generate the bit
2838 // encoding of the instruction. Operands specify their base encoding
2839 // interface with the interface keyword. There are currently
2840 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2841 // COND_INTER. REG_INTER causes an operand to generate a function
2842 // which returns its register number when queried. CONST_INTER causes
2843 // an operand to generate a function which returns the value of the
2844 // constant when queried. MEMORY_INTER causes an operand to generate
2845 // four functions which return the Base Register, the Index Register,
2846 // the Scale Value, and the Offset Value of the operand when queried.
2847 // COND_INTER causes an operand to generate six functions which return
2848 // the encoding code (ie - encoding bits for the instruction)
2849 // associated with each basic boolean condition for a conditional
2850 // instruction.
2851 //
2852 // Instructions specify two basic values for encoding. Again, a
2853 // function is available to check if the constant displacement is an
2854 // oop. They use the ins_encode keyword to specify their encoding
2855 // classes (which must be a sequence of enc_class names, and their
2856 // parameters, specified in the encoding block), and they use the
2857 // opcode keyword to specify, in order, their primary, secondary, and
2858 // tertiary opcode. Only the opcode sections which a particular
2859 // instruction needs for encoding need to be specified.
2860 encode %{
2861 // Build emit functions for each basic byte or larger field in the
2862 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2863 // from C++ code in the enc_class source block. Emit functions will
2864 // live in the main source block for now. In future, we can
2865 // generalize this by adding a syntax that specifies the sizes of
2866 // fields in an order, so that the adlc can build the emit functions
2867 // automagically
2868
2869 // catch all for unimplemented encodings
2870 enc_class enc_unimplemented %{
2871 __ unimplemented("C2 catch all");
2872 %}
2873
2874 // BEGIN Non-volatile memory access
2875
2876 // This encoding class is generated automatically from ad_encode.m4.
2877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2878 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2879 Register dst_reg = as_Register($dst$$reg);
2880 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2881 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2882 %}
2883
2884 // This encoding class is generated automatically from ad_encode.m4.
2885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2886 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2887 Register dst_reg = as_Register($dst$$reg);
2888 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2889 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2890 %}
2891
2892 // This encoding class is generated automatically from ad_encode.m4.
2893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2894 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2895 Register dst_reg = as_Register($dst$$reg);
2896 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2897 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2898 %}
2899
2900 // This encoding class is generated automatically from ad_encode.m4.
2901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2902 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2903 Register dst_reg = as_Register($dst$$reg);
2904 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2905 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2906 %}
2907
2908 // This encoding class is generated automatically from ad_encode.m4.
2909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2910 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2911 Register dst_reg = as_Register($dst$$reg);
2912 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2913 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2914 %}
2915
2916 // This encoding class is generated automatically from ad_encode.m4.
2917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2918 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2919 Register dst_reg = as_Register($dst$$reg);
2920 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2921 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2922 %}
2923
2924 // This encoding class is generated automatically from ad_encode.m4.
2925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2926 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2927 Register dst_reg = as_Register($dst$$reg);
2928 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2929 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2930 %}
2931
2932 // This encoding class is generated automatically from ad_encode.m4.
2933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2934 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2935 Register dst_reg = as_Register($dst$$reg);
2936 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2937 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2938 %}
2939
2940 // This encoding class is generated automatically from ad_encode.m4.
2941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2942 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2943 Register dst_reg = as_Register($dst$$reg);
2944 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2945 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2946 %}
2947
2948 // This encoding class is generated automatically from ad_encode.m4.
2949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2950 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2951 Register dst_reg = as_Register($dst$$reg);
2952 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2953 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2954 %}
2955
2956 // This encoding class is generated automatically from ad_encode.m4.
2957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2958 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2959 Register dst_reg = as_Register($dst$$reg);
2960 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2961 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2962 %}
2963
2964 // This encoding class is generated automatically from ad_encode.m4.
2965 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2966 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2967 Register dst_reg = as_Register($dst$$reg);
2968 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2969 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2970 %}
2971
2972 // This encoding class is generated automatically from ad_encode.m4.
2973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2974 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2975 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2976 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2977 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2978 %}
2979
2980 // This encoding class is generated automatically from ad_encode.m4.
2981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2982 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2983 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2984 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2985 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2986 %}
2987
2988 // This encoding class is generated automatically from ad_encode.m4.
2989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2990 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2991 Register src_reg = as_Register($src$$reg);
2992 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2993 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2994 %}
2995
2996 // This encoding class is generated automatically from ad_encode.m4.
2997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2998 enc_class aarch64_enc_strb0(memory1 mem) %{
2999 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3000 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3001 %}
3002
3003 // This encoding class is generated automatically from ad_encode.m4.
3004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3005 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3006 Register src_reg = as_Register($src$$reg);
3007 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3008 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3009 %}
3010
3011 // This encoding class is generated automatically from ad_encode.m4.
3012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3013 enc_class aarch64_enc_strh0(memory2 mem) %{
3014 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3015 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3016 %}
3017
3018 // This encoding class is generated automatically from ad_encode.m4.
3019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3020 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3021 Register src_reg = as_Register($src$$reg);
3022 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_strw0(memory4 mem) %{
3029 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3030 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3031 %}
3032
3033 // This encoding class is generated automatically from ad_encode.m4.
3034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3035 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3036 Register src_reg = as_Register($src$$reg);
3037 // we sometimes get asked to store the stack pointer into the
3038 // current thread -- we cannot do that directly on AArch64
3039 if (src_reg == r31_sp) {
3040 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3041 __ mov(rscratch2, sp);
3042 src_reg = rscratch2;
3043 }
3044 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3045 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3046 %}
3047
3048 // This encoding class is generated automatically from ad_encode.m4.
3049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3050 enc_class aarch64_enc_str0(memory8 mem) %{
3051 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3052 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3053 %}
3054
3055 // This encoding class is generated automatically from ad_encode.m4.
3056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3057 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3058 FloatRegister src_reg = as_FloatRegister($src$$reg);
3059 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3060 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3061 %}
3062
3063 // This encoding class is generated automatically from ad_encode.m4.
3064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3065 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3066 FloatRegister src_reg = as_FloatRegister($src$$reg);
3067 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3068 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3069 %}
3070
3071 // This encoding class is generated automatically from ad_encode.m4.
3072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3073 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3074 __ membar(Assembler::StoreStore);
3075 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3076 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3077 %}
3078
3079 // END Non-volatile memory access
3080
3081 // Vector loads and stores
3082 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3083 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3084 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3085 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3086 %}
3087
3088 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3089 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3090 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3091 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3092 %}
3093
3094 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3095 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3096 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3097 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3098 %}
3099
3100 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3101 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3102 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3103 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3104 %}
3105
3106 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3107 FloatRegister src_reg = as_FloatRegister($src$$reg);
3108 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3109 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3110 %}
3111
3112 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3113 FloatRegister src_reg = as_FloatRegister($src$$reg);
3114 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3115 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3116 %}
3117
3118 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3119 FloatRegister src_reg = as_FloatRegister($src$$reg);
3120 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3121 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3122 %}
3123
3124 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3125 FloatRegister src_reg = as_FloatRegister($src$$reg);
3126 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3127 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3128 %}
3129
3130 // volatile loads and stores
3131
3132 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3133 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3134 rscratch1, stlrb);
3135 %}
3136
3137 enc_class aarch64_enc_stlrb0(memory mem) %{
3138 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3139 rscratch1, stlrb);
3140 %}
3141
3142 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3143 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3144 rscratch1, stlrh);
3145 %}
3146
3147 enc_class aarch64_enc_stlrh0(memory mem) %{
3148 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, stlrh);
3150 %}
3151
3152 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3153 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, stlrw);
3155 %}
3156
3157 enc_class aarch64_enc_stlrw0(memory mem) %{
3158 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3159 rscratch1, stlrw);
3160 %}
3161
3162 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3163 Register dst_reg = as_Register($dst$$reg);
3164 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3165 rscratch1, ldarb);
3166 __ sxtbw(dst_reg, dst_reg);
3167 %}
3168
3169 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3170 Register dst_reg = as_Register($dst$$reg);
3171 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3172 rscratch1, ldarb);
3173 __ sxtb(dst_reg, dst_reg);
3174 %}
3175
3176 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3177 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3178 rscratch1, ldarb);
3179 %}
3180
3181 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3182 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3183 rscratch1, ldarb);
3184 %}
3185
3186 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3187 Register dst_reg = as_Register($dst$$reg);
3188 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, ldarh);
3190 __ sxthw(dst_reg, dst_reg);
3191 %}
3192
3193 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3194 Register dst_reg = as_Register($dst$$reg);
3195 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3196 rscratch1, ldarh);
3197 __ sxth(dst_reg, dst_reg);
3198 %}
3199
3200 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3201 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3202 rscratch1, ldarh);
3203 %}
3204
3205 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3206 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3207 rscratch1, ldarh);
3208 %}
3209
3210 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3211 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, ldarw);
3213 %}
3214
3215 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3216 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3217 rscratch1, ldarw);
3218 %}
3219
3220 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3221 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3222 rscratch1, ldar);
3223 %}
3224
3225 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3226 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3227 rscratch1, ldarw);
3228 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3229 %}
3230
3231 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3232 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3233 rscratch1, ldar);
3234 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3235 %}
3236
3237 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3238 Register src_reg = as_Register($src$$reg);
3239 // we sometimes get asked to store the stack pointer into the
3240 // current thread -- we cannot do that directly on AArch64
3241 if (src_reg == r31_sp) {
3242 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3243 __ mov(rscratch2, sp);
3244 src_reg = rscratch2;
3245 }
3246 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3247 rscratch1, stlr);
3248 %}
3249
3250 enc_class aarch64_enc_stlr0(memory mem) %{
3251 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3252 rscratch1, stlr);
3253 %}
3254
3255 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3256 {
3257 FloatRegister src_reg = as_FloatRegister($src$$reg);
3258 __ fmovs(rscratch2, src_reg);
3259 }
3260 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3261 rscratch1, stlrw);
3262 %}
3263
3264 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3265 {
3266 FloatRegister src_reg = as_FloatRegister($src$$reg);
3267 __ fmovd(rscratch2, src_reg);
3268 }
3269 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3270 rscratch1, stlr);
3271 %}
3272
3273 // synchronized read/update encodings
3274
3275 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3276 Register dst_reg = as_Register($dst$$reg);
3277 Register base = as_Register($mem$$base);
3278 int index = $mem$$index;
3279 int scale = $mem$$scale;
3280 int disp = $mem$$disp;
3281 if (index == -1) {
3282 if (disp != 0) {
3283 __ lea(rscratch1, Address(base, disp));
3284 __ ldaxr(dst_reg, rscratch1);
3285 } else {
3286 // TODO
3287 // should we ever get anything other than this case?
3288 __ ldaxr(dst_reg, base);
3289 }
3290 } else {
3291 Register index_reg = as_Register(index);
3292 if (disp == 0) {
3293 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3294 __ ldaxr(dst_reg, rscratch1);
3295 } else {
3296 __ lea(rscratch1, Address(base, disp));
3297 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3298 __ ldaxr(dst_reg, rscratch1);
3299 }
3300 }
3301 %}
3302
3303 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3304 Register src_reg = as_Register($src$$reg);
3305 Register base = as_Register($mem$$base);
3306 int index = $mem$$index;
3307 int scale = $mem$$scale;
3308 int disp = $mem$$disp;
3309 if (index == -1) {
3310 if (disp != 0) {
3311 __ lea(rscratch2, Address(base, disp));
3312 __ stlxr(rscratch1, src_reg, rscratch2);
3313 } else {
3314 // TODO
3315 // should we ever get anything other than this case?
3316 __ stlxr(rscratch1, src_reg, base);
3317 }
3318 } else {
3319 Register index_reg = as_Register(index);
3320 if (disp == 0) {
3321 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3322 __ stlxr(rscratch1, src_reg, rscratch2);
3323 } else {
3324 __ lea(rscratch2, Address(base, disp));
3325 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3326 __ stlxr(rscratch1, src_reg, rscratch2);
3327 }
3328 }
3329 __ cmpw(rscratch1, zr);
3330 %}
3331
3332 // prefetch encodings
3333
3334 enc_class aarch64_enc_prefetchw(memory mem) %{
3335 Register base = as_Register($mem$$base);
3336 int index = $mem$$index;
3337 int scale = $mem$$scale;
3338 int disp = $mem$$disp;
3339 if (index == -1) {
3340 // Fix up any out-of-range offsets.
3341 assert_different_registers(rscratch1, base);
3342 Address addr = Address(base, disp);
3343 addr = __ legitimize_address(addr, 8, rscratch1);
3344 __ prfm(addr, PSTL1KEEP);
3345 } else {
3346 Register index_reg = as_Register(index);
3347 if (disp == 0) {
3348 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3349 } else {
3350 __ lea(rscratch1, Address(base, disp));
3351 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3352 }
3353 }
3354 %}
3355
3356 // mov encodings
3357
3358 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3359 uint32_t con = (uint32_t)$src$$constant;
3360 Register dst_reg = as_Register($dst$$reg);
3361 if (con == 0) {
3362 __ movw(dst_reg, zr);
3363 } else {
3364 __ movw(dst_reg, con);
3365 }
3366 %}
3367
3368 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3369 Register dst_reg = as_Register($dst$$reg);
3370 uint64_t con = (uint64_t)$src$$constant;
3371 if (con == 0) {
3372 __ mov(dst_reg, zr);
3373 } else {
3374 __ mov(dst_reg, con);
3375 }
3376 %}
3377
3378 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3379 Register dst_reg = as_Register($dst$$reg);
3380 address con = (address)$src$$constant;
3381 if (con == nullptr || con == (address)1) {
3382 ShouldNotReachHere();
3383 } else {
3384 relocInfo::relocType rtype = $src->constant_reloc();
3385 if (rtype == relocInfo::oop_type) {
3386 __ movoop(dst_reg, (jobject)con);
3387 } else if (rtype == relocInfo::metadata_type) {
3388 __ mov_metadata(dst_reg, (Metadata*)con);
3389 } else {
3390 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3391 // load fake address constants using a normal move
3392 if (! __ is_valid_AArch64_address(con) ||
3393 con < (address)(uintptr_t)os::vm_page_size()) {
3394 __ mov(dst_reg, con);
3395 } else {
3396 // no reloc so just use adrp and add
3397 uint64_t offset;
3398 __ adrp(dst_reg, con, offset);
3399 __ add(dst_reg, dst_reg, offset);
3400 }
3401 }
3402 }
3403 %}
3404
3405 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3406 Register dst_reg = as_Register($dst$$reg);
3407 __ mov(dst_reg, zr);
3408 %}
3409
3410 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3411 Register dst_reg = as_Register($dst$$reg);
3412 __ mov(dst_reg, (uint64_t)1);
3413 %}
3414
3415 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3416 Register dst_reg = as_Register($dst$$reg);
3417 address con = (address)$src$$constant;
3418 if (con == nullptr) {
3419 ShouldNotReachHere();
3420 } else {
3421 relocInfo::relocType rtype = $src->constant_reloc();
3422 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3423 __ set_narrow_oop(dst_reg, (jobject)con);
3424 }
3425 %}
3426
3427 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3428 Register dst_reg = as_Register($dst$$reg);
3429 __ mov(dst_reg, zr);
3430 %}
3431
3432 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3433 Register dst_reg = as_Register($dst$$reg);
3434 address con = (address)$src$$constant;
3435 if (con == nullptr) {
3436 ShouldNotReachHere();
3437 } else {
3438 relocInfo::relocType rtype = $src->constant_reloc();
3439 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3440 __ set_narrow_klass(dst_reg, (Klass *)con);
3441 }
3442 %}
3443
3444 // arithmetic encodings
3445
3446 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3447 Register dst_reg = as_Register($dst$$reg);
3448 Register src_reg = as_Register($src1$$reg);
3449 int32_t con = (int32_t)$src2$$constant;
3450 // add has primary == 0, subtract has primary == 1
3451 if ($primary) { con = -con; }
3452 if (con < 0) {
3453 __ subw(dst_reg, src_reg, -con);
3454 } else {
3455 __ addw(dst_reg, src_reg, con);
3456 }
3457 %}
3458
3459 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3460 Register dst_reg = as_Register($dst$$reg);
3461 Register src_reg = as_Register($src1$$reg);
3462 int32_t con = (int32_t)$src2$$constant;
3463 // add has primary == 0, subtract has primary == 1
3464 if ($primary) { con = -con; }
3465 if (con < 0) {
3466 __ sub(dst_reg, src_reg, -con);
3467 } else {
3468 __ add(dst_reg, src_reg, con);
3469 }
3470 %}
3471
3472 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3473 Register dst_reg = as_Register($dst$$reg);
3474 Register src1_reg = as_Register($src1$$reg);
3475 Register src2_reg = as_Register($src2$$reg);
3476 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3477 %}
3478
3479 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 Register src1_reg = as_Register($src1$$reg);
3482 Register src2_reg = as_Register($src2$$reg);
3483 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3484 %}
3485
3486 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3487 Register dst_reg = as_Register($dst$$reg);
3488 Register src1_reg = as_Register($src1$$reg);
3489 Register src2_reg = as_Register($src2$$reg);
3490 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3491 %}
3492
3493 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3494 Register dst_reg = as_Register($dst$$reg);
3495 Register src1_reg = as_Register($src1$$reg);
3496 Register src2_reg = as_Register($src2$$reg);
3497 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3498 %}
3499
3500 // compare instruction encodings
3501
3502 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3503 Register reg1 = as_Register($src1$$reg);
3504 Register reg2 = as_Register($src2$$reg);
3505 __ cmpw(reg1, reg2);
3506 %}
3507
3508 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3509 Register reg = as_Register($src1$$reg);
3510 int32_t val = $src2$$constant;
3511 if (val >= 0) {
3512 __ subsw(zr, reg, val);
3513 } else {
3514 __ addsw(zr, reg, -val);
3515 }
3516 %}
3517
3518 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3519 Register reg1 = as_Register($src1$$reg);
3520 uint32_t val = (uint32_t)$src2$$constant;
3521 __ movw(rscratch1, val);
3522 __ cmpw(reg1, rscratch1);
3523 %}
3524
3525 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3526 Register reg1 = as_Register($src1$$reg);
3527 Register reg2 = as_Register($src2$$reg);
3528 __ cmp(reg1, reg2);
3529 %}
3530
3531 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3532 Register reg = as_Register($src1$$reg);
3533 int64_t val = $src2$$constant;
3534 if (val >= 0) {
3535 __ subs(zr, reg, val);
3536 } else if (val != -val) {
3537 __ adds(zr, reg, -val);
3538 } else {
3539 // aargh, Long.MIN_VALUE is a special case
3540 __ orr(rscratch1, zr, (uint64_t)val);
3541 __ subs(zr, reg, rscratch1);
3542 }
3543 %}
3544
3545 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3546 Register reg1 = as_Register($src1$$reg);
3547 uint64_t val = (uint64_t)$src2$$constant;
3548 __ mov(rscratch1, val);
3549 __ cmp(reg1, rscratch1);
3550 %}
3551
3552 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3553 Register reg1 = as_Register($src1$$reg);
3554 Register reg2 = as_Register($src2$$reg);
3555 __ cmp(reg1, reg2);
3556 %}
3557
3558 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3559 Register reg1 = as_Register($src1$$reg);
3560 Register reg2 = as_Register($src2$$reg);
3561 __ cmpw(reg1, reg2);
3562 %}
3563
3564 enc_class aarch64_enc_testp(iRegP src) %{
3565 Register reg = as_Register($src$$reg);
3566 __ cmp(reg, zr);
3567 %}
3568
3569 enc_class aarch64_enc_testn(iRegN src) %{
3570 Register reg = as_Register($src$$reg);
3571 __ cmpw(reg, zr);
3572 %}
3573
3574 enc_class aarch64_enc_b(label lbl) %{
3575 Label *L = $lbl$$label;
3576 __ b(*L);
3577 %}
3578
3579 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3580 Label *L = $lbl$$label;
3581 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3582 %}
3583
3584 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3585 Label *L = $lbl$$label;
3586 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3587 %}
3588
3589 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3590 %{
3591 Register sub_reg = as_Register($sub$$reg);
3592 Register super_reg = as_Register($super$$reg);
3593 Register temp_reg = as_Register($temp$$reg);
3594 Register result_reg = as_Register($result$$reg);
3595
3596 Label miss;
3597 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3598 nullptr, &miss,
3599 /*set_cond_codes:*/ true);
3600 if ($primary) {
3601 __ mov(result_reg, zr);
3602 }
3603 __ bind(miss);
3604 %}
3605
3606 enc_class aarch64_enc_java_static_call(method meth) %{
3607 address addr = (address)$meth$$method;
3608 address call;
3609 if (!_method) {
3610 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3611 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3612 if (call == nullptr) {
3613 ciEnv::current()->record_failure("CodeCache is full");
3614 return;
3615 }
3616 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3617 // The NOP here is purely to ensure that eliding a call to
3618 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3619 __ nop();
3620 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3621 } else {
3622 int method_index = resolved_method_index(masm);
3623 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3624 : static_call_Relocation::spec(method_index);
3625 call = __ trampoline_call(Address(addr, rspec));
3626 if (call == nullptr) {
3627 ciEnv::current()->record_failure("CodeCache is full");
3628 return;
3629 }
3630 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3631 // Calls of the same statically bound method can share
3632 // a stub to the interpreter.
3633 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3634 } else {
3635 // Emit stub for static call
3636 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3637 if (stub == nullptr) {
3638 ciEnv::current()->record_failure("CodeCache is full");
3639 return;
3640 }
3641 }
3642 }
3643
3644 __ post_call_nop();
3645
3646 // Only non uncommon_trap calls need to reinitialize ptrue.
3647 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3648 __ reinitialize_ptrue();
3649 }
3650 %}
3651
3652 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3653 int method_index = resolved_method_index(masm);
3654 address call = __ ic_call((address)$meth$$method, method_index);
3655 if (call == nullptr) {
3656 ciEnv::current()->record_failure("CodeCache is full");
3657 return;
3658 }
3659 __ post_call_nop();
3660 if (Compile::current()->max_vector_size() > 0) {
3661 __ reinitialize_ptrue();
3662 }
3663 %}
3664
3665 enc_class aarch64_enc_call_epilog() %{
3666 if (VerifyStackAtCalls) {
3667 // Check that stack depth is unchanged: find majik cookie on stack
3668 __ call_Unimplemented();
3669 }
3670 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3671 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3672 // Search for the corresponding projection, get the register and emit code that initialized it.
3673 uint con = (tf()->range_cc()->cnt() - 1);
3674 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3675 ProjNode* proj = fast_out(i)->as_Proj();
3676 if (proj->_con == con) {
3677 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3678 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3679 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3680 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3681 __ cmp(r0, zr);
3682 __ cset(toReg, Assembler::NE);
3683 if (reg->is_stack()) {
3684 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3685 __ str(toReg, Address(sp, st_off));
3686 }
3687 break;
3688 }
3689 }
3690 if (return_value_is_used()) {
3691 // An inline type is returned as fields in multiple registers.
3692 // R0 either contains an oop if the inline type is buffered or a pointer
3693 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3694 // if the lowest bit is set to allow C2 to use the oop after null checking.
3695 // r0 &= (r0 & 1) - 1
3696 __ andr(rscratch1, r0, 0x1);
3697 __ sub(rscratch1, rscratch1, 0x1);
3698 __ andr(r0, r0, rscratch1);
3699 }
3700 }
3701 %}
3702
3703 enc_class aarch64_enc_java_to_runtime(method meth) %{
3704 // some calls to generated routines (arraycopy code) are scheduled
3705 // by C2 as runtime calls. if so we can call them using a br (they
3706 // will be in a reachable segment) otherwise we have to use a blr
3707 // which loads the absolute address into a register.
3708 address entry = (address)$meth$$method;
3709 CodeBlob *cb = CodeCache::find_blob(entry);
3710 if (cb) {
3711 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3712 if (call == nullptr) {
3713 ciEnv::current()->record_failure("CodeCache is full");
3714 return;
3715 }
3716 __ post_call_nop();
3717 } else {
3718 Label retaddr;
3719 // Make the anchor frame walkable
3720 __ adr(rscratch2, retaddr);
3721 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3722 __ lea(rscratch1, RuntimeAddress(entry));
3723 __ blr(rscratch1);
3724 __ bind(retaddr);
3725 __ post_call_nop();
3726 }
3727 if (Compile::current()->max_vector_size() > 0) {
3728 __ reinitialize_ptrue();
3729 }
3730 %}
3731
3732 enc_class aarch64_enc_rethrow() %{
3733 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3734 %}
3735
3736 enc_class aarch64_enc_ret() %{
3737 #ifdef ASSERT
3738 if (Compile::current()->max_vector_size() > 0) {
3739 __ verify_ptrue();
3740 }
3741 #endif
3742 __ ret(lr);
3743 %}
3744
3745 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3746 Register target_reg = as_Register($jump_target$$reg);
3747 __ br(target_reg);
3748 %}
3749
3750 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3751 Register target_reg = as_Register($jump_target$$reg);
3752 // exception oop should be in r0
3753 // ret addr has been popped into lr
3754 // callee expects it in r3
3755 __ mov(r3, lr);
3756 __ br(target_reg);
3757 %}
3758
3759 %}
3760
3761 //----------FRAME--------------------------------------------------------------
3762 // Definition of frame structure and management information.
3763 //
3764 // S T A C K L A Y O U T Allocators stack-slot number
3765 // | (to get allocators register number
3766 // G Owned by | | v add OptoReg::stack0())
3767 // r CALLER | |
3768 // o | +--------+ pad to even-align allocators stack-slot
3769 // w V | pad0 | numbers; owned by CALLER
3770 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3771 // h ^ | in | 5
3772 // | | args | 4 Holes in incoming args owned by SELF
3773 // | | | | 3
3774 // | | +--------+
3775 // V | | old out| Empty on Intel, window on Sparc
3776 // | old |preserve| Must be even aligned.
3777 // | SP-+--------+----> Matcher::_old_SP, even aligned
3778 // | | in | 3 area for Intel ret address
3779 // Owned by |preserve| Empty on Sparc.
3780 // SELF +--------+
3781 // | | pad2 | 2 pad to align old SP
3782 // | +--------+ 1
3783 // | | locks | 0
3784 // | +--------+----> OptoReg::stack0(), even aligned
3785 // | | pad1 | 11 pad to align new SP
3786 // | +--------+
3787 // | | | 10
3788 // | | spills | 9 spills
3789 // V | | 8 (pad0 slot for callee)
3790 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3791 // ^ | out | 7
3792 // | | args | 6 Holes in outgoing args owned by CALLEE
3793 // Owned by +--------+
3794 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3795 // | new |preserve| Must be even-aligned.
3796 // | SP-+--------+----> Matcher::_new_SP, even aligned
3797 // | | |
3798 //
3799 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3800 // known from SELF's arguments and the Java calling convention.
3801 // Region 6-7 is determined per call site.
3802 // Note 2: If the calling convention leaves holes in the incoming argument
3803 // area, those holes are owned by SELF. Holes in the outgoing area
3804 // are owned by the CALLEE. Holes should not be necessary in the
3805 // incoming area, as the Java calling convention is completely under
3806 // the control of the AD file. Doubles can be sorted and packed to
3807 // avoid holes. Holes in the outgoing arguments may be necessary for
3808 // varargs C calling conventions.
3809 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3810 // even aligned with pad0 as needed.
3811 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3812 // (the latter is true on Intel but is it false on AArch64?)
3813 // region 6-11 is even aligned; it may be padded out more so that
3814 // the region from SP to FP meets the minimum stack alignment.
3815 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3816 // alignment. Region 11, pad1, may be dynamically extended so that
3817 // SP meets the minimum alignment.
3818
3819 frame %{
3820 // These three registers define part of the calling convention
3821 // between compiled code and the interpreter.
3822
3823 // Inline Cache Register or Method for I2C.
3824 inline_cache_reg(R12);
3825
3826 // Number of stack slots consumed by locking an object
3827 sync_stack_slots(2);
3828
3829 // Compiled code's Frame Pointer
3830 frame_pointer(R31);
3831
3832 // Stack alignment requirement
3833 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3834
3835 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3836 // for calls to C. Supports the var-args backing area for register parms.
3837 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3838
3839 // The after-PROLOG location of the return address. Location of
3840 // return address specifies a type (REG or STACK) and a number
3841 // representing the register number (i.e. - use a register name) or
3842 // stack slot.
3843 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3844 // Otherwise, it is above the locks and verification slot and alignment word
3845 // TODO this may well be correct but need to check why that - 2 is there
3846 // ppc port uses 0 but we definitely need to allow for fixed_slots
3847 // which folds in the space used for monitors
3848 return_addr(STACK - 2 +
3849 align_up((Compile::current()->in_preserve_stack_slots() +
3850 Compile::current()->fixed_slots()),
3851 stack_alignment_in_slots()));
3852
3853 // Location of compiled Java return values. Same as C for now.
3854 return_value
3855 %{
3856 // TODO do we allow ideal_reg == Op_RegN???
3857 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3858 "only return normal values");
3859
3860 static const int lo[Op_RegL + 1] = { // enum name
3861 0, // Op_Node
3862 0, // Op_Set
3863 R0_num, // Op_RegN
3864 R0_num, // Op_RegI
3865 R0_num, // Op_RegP
3866 V0_num, // Op_RegF
3867 V0_num, // Op_RegD
3868 R0_num // Op_RegL
3869 };
3870
3871 static const int hi[Op_RegL + 1] = { // enum name
3872 0, // Op_Node
3873 0, // Op_Set
3874 OptoReg::Bad, // Op_RegN
3875 OptoReg::Bad, // Op_RegI
3876 R0_H_num, // Op_RegP
3877 OptoReg::Bad, // Op_RegF
3878 V0_H_num, // Op_RegD
3879 R0_H_num // Op_RegL
3880 };
3881
3882 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3883 %}
3884 %}
3885
3886 //----------ATTRIBUTES---------------------------------------------------------
3887 //----------Operand Attributes-------------------------------------------------
3888 op_attrib op_cost(1); // Required cost attribute
3889
3890 //----------Instruction Attributes---------------------------------------------
3891 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3892 ins_attrib ins_size(32); // Required size attribute (in bits)
3893 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3894 // a non-matching short branch variant
3895 // of some long branch?
3896 ins_attrib ins_alignment(4); // Required alignment attribute (must
3897 // be a power of 2) specifies the
3898 // alignment that some part of the
3899 // instruction (not necessarily the
3900 // start) requires. If > 1, a
3901 // compute_padding() function must be
3902 // provided for the instruction
3903
3904 // Whether this node is expanded during code emission into a sequence of
3905 // instructions and the first instruction can perform an implicit null check.
3906 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3907
3908 //----------OPERANDS-----------------------------------------------------------
3909 // Operand definitions must precede instruction definitions for correct parsing
3910 // in the ADLC because operands constitute user defined types which are used in
3911 // instruction definitions.
3912
3913 //----------Simple Operands----------------------------------------------------
3914
3915 // Integer operands 32 bit
3916 // 32 bit immediate
3917 operand immI()
3918 %{
3919 match(ConI);
3920
3921 op_cost(0);
3922 format %{ %}
3923 interface(CONST_INTER);
3924 %}
3925
3926 // 32 bit zero
3927 operand immI0()
3928 %{
3929 predicate(n->get_int() == 0);
3930 match(ConI);
3931
3932 op_cost(0);
3933 format %{ %}
3934 interface(CONST_INTER);
3935 %}
3936
3937 // 32 bit unit increment
3938 operand immI_1()
3939 %{
3940 predicate(n->get_int() == 1);
3941 match(ConI);
3942
3943 op_cost(0);
3944 format %{ %}
3945 interface(CONST_INTER);
3946 %}
3947
3948 // 32 bit unit decrement
3949 operand immI_M1()
3950 %{
3951 predicate(n->get_int() == -1);
3952 match(ConI);
3953
3954 op_cost(0);
3955 format %{ %}
3956 interface(CONST_INTER);
3957 %}
3958
3959 // Shift values for add/sub extension shift
3960 operand immIExt()
3961 %{
3962 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3963 match(ConI);
3964
3965 op_cost(0);
3966 format %{ %}
3967 interface(CONST_INTER);
3968 %}
3969
3970 operand immI_gt_1()
3971 %{
3972 predicate(n->get_int() > 1);
3973 match(ConI);
3974
3975 op_cost(0);
3976 format %{ %}
3977 interface(CONST_INTER);
3978 %}
3979
3980 operand immI_le_4()
3981 %{
3982 predicate(n->get_int() <= 4);
3983 match(ConI);
3984
3985 op_cost(0);
3986 format %{ %}
3987 interface(CONST_INTER);
3988 %}
3989
3990 operand immI_4()
3991 %{
3992 predicate(n->get_int() == 4);
3993 match(ConI);
3994
3995 op_cost(0);
3996 format %{ %}
3997 interface(CONST_INTER);
3998 %}
3999
4000 operand immI_16()
4001 %{
4002 predicate(n->get_int() == 16);
4003 match(ConI);
4004
4005 op_cost(0);
4006 format %{ %}
4007 interface(CONST_INTER);
4008 %}
4009
4010 operand immI_24()
4011 %{
4012 predicate(n->get_int() == 24);
4013 match(ConI);
4014
4015 op_cost(0);
4016 format %{ %}
4017 interface(CONST_INTER);
4018 %}
4019
4020 operand immI_32()
4021 %{
4022 predicate(n->get_int() == 32);
4023 match(ConI);
4024
4025 op_cost(0);
4026 format %{ %}
4027 interface(CONST_INTER);
4028 %}
4029
4030 operand immI_48()
4031 %{
4032 predicate(n->get_int() == 48);
4033 match(ConI);
4034
4035 op_cost(0);
4036 format %{ %}
4037 interface(CONST_INTER);
4038 %}
4039
4040 operand immI_56()
4041 %{
4042 predicate(n->get_int() == 56);
4043 match(ConI);
4044
4045 op_cost(0);
4046 format %{ %}
4047 interface(CONST_INTER);
4048 %}
4049
4050 operand immI_255()
4051 %{
4052 predicate(n->get_int() == 255);
4053 match(ConI);
4054
4055 op_cost(0);
4056 format %{ %}
4057 interface(CONST_INTER);
4058 %}
4059
4060 operand immI_65535()
4061 %{
4062 predicate(n->get_int() == 65535);
4063 match(ConI);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 operand immI_positive()
4071 %{
4072 predicate(n->get_int() > 0);
4073 match(ConI);
4074
4075 op_cost(0);
4076 format %{ %}
4077 interface(CONST_INTER);
4078 %}
4079
4080 // BoolTest condition for signed compare
4081 operand immI_cmp_cond()
4082 %{
4083 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4084 match(ConI);
4085
4086 op_cost(0);
4087 format %{ %}
4088 interface(CONST_INTER);
4089 %}
4090
4091 // BoolTest condition for unsigned compare
4092 operand immI_cmpU_cond()
4093 %{
4094 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4095 match(ConI);
4096
4097 op_cost(0);
4098 format %{ %}
4099 interface(CONST_INTER);
4100 %}
4101
4102 operand immL_255()
4103 %{
4104 predicate(n->get_long() == 255L);
4105 match(ConL);
4106
4107 op_cost(0);
4108 format %{ %}
4109 interface(CONST_INTER);
4110 %}
4111
4112 operand immL_65535()
4113 %{
4114 predicate(n->get_long() == 65535L);
4115 match(ConL);
4116
4117 op_cost(0);
4118 format %{ %}
4119 interface(CONST_INTER);
4120 %}
4121
4122 operand immL_4294967295()
4123 %{
4124 predicate(n->get_long() == 4294967295L);
4125 match(ConL);
4126
4127 op_cost(0);
4128 format %{ %}
4129 interface(CONST_INTER);
4130 %}
4131
4132 operand immL_bitmask()
4133 %{
4134 predicate((n->get_long() != 0)
4135 && ((n->get_long() & 0xc000000000000000l) == 0)
4136 && is_power_of_2(n->get_long() + 1));
4137 match(ConL);
4138
4139 op_cost(0);
4140 format %{ %}
4141 interface(CONST_INTER);
4142 %}
4143
4144 operand immI_bitmask()
4145 %{
4146 predicate((n->get_int() != 0)
4147 && ((n->get_int() & 0xc0000000) == 0)
4148 && is_power_of_2(n->get_int() + 1));
4149 match(ConI);
4150
4151 op_cost(0);
4152 format %{ %}
4153 interface(CONST_INTER);
4154 %}
4155
4156 operand immL_positive_bitmaskI()
4157 %{
4158 predicate((n->get_long() != 0)
4159 && ((julong)n->get_long() < 0x80000000ULL)
4160 && is_power_of_2(n->get_long() + 1));
4161 match(ConL);
4162
4163 op_cost(0);
4164 format %{ %}
4165 interface(CONST_INTER);
4166 %}
4167
4168 // Scale values for scaled offset addressing modes (up to long but not quad)
4169 operand immIScale()
4170 %{
4171 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4172 match(ConI);
4173
4174 op_cost(0);
4175 format %{ %}
4176 interface(CONST_INTER);
4177 %}
4178
4179 // 5 bit signed integer
4180 operand immI5()
4181 %{
4182 predicate(Assembler::is_simm(n->get_int(), 5));
4183 match(ConI);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 // 7 bit unsigned integer
4191 operand immIU7()
4192 %{
4193 predicate(Assembler::is_uimm(n->get_int(), 7));
4194 match(ConI);
4195
4196 op_cost(0);
4197 format %{ %}
4198 interface(CONST_INTER);
4199 %}
4200
4201 // Offset for scaled or unscaled immediate loads and stores
4202 operand immIOffset()
4203 %{
4204 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4205 match(ConI);
4206
4207 op_cost(0);
4208 format %{ %}
4209 interface(CONST_INTER);
4210 %}
4211
4212 operand immIOffset1()
4213 %{
4214 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4215 match(ConI);
4216
4217 op_cost(0);
4218 format %{ %}
4219 interface(CONST_INTER);
4220 %}
4221
4222 operand immIOffset2()
4223 %{
4224 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4225 match(ConI);
4226
4227 op_cost(0);
4228 format %{ %}
4229 interface(CONST_INTER);
4230 %}
4231
4232 operand immIOffset4()
4233 %{
4234 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4235 match(ConI);
4236
4237 op_cost(0);
4238 format %{ %}
4239 interface(CONST_INTER);
4240 %}
4241
4242 operand immIOffset8()
4243 %{
4244 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4245 match(ConI);
4246
4247 op_cost(0);
4248 format %{ %}
4249 interface(CONST_INTER);
4250 %}
4251
4252 operand immIOffset16()
4253 %{
4254 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4255 match(ConI);
4256
4257 op_cost(0);
4258 format %{ %}
4259 interface(CONST_INTER);
4260 %}
4261
4262 operand immLOffset()
4263 %{
4264 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4265 match(ConL);
4266
4267 op_cost(0);
4268 format %{ %}
4269 interface(CONST_INTER);
4270 %}
4271
4272 operand immLoffset1()
4273 %{
4274 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4275 match(ConL);
4276
4277 op_cost(0);
4278 format %{ %}
4279 interface(CONST_INTER);
4280 %}
4281
4282 operand immLoffset2()
4283 %{
4284 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4285 match(ConL);
4286
4287 op_cost(0);
4288 format %{ %}
4289 interface(CONST_INTER);
4290 %}
4291
4292 operand immLoffset4()
4293 %{
4294 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4295 match(ConL);
4296
4297 op_cost(0);
4298 format %{ %}
4299 interface(CONST_INTER);
4300 %}
4301
4302 operand immLoffset8()
4303 %{
4304 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4305 match(ConL);
4306
4307 op_cost(0);
4308 format %{ %}
4309 interface(CONST_INTER);
4310 %}
4311
4312 operand immLoffset16()
4313 %{
4314 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4315 match(ConL);
4316
4317 op_cost(0);
4318 format %{ %}
4319 interface(CONST_INTER);
4320 %}
4321
4322 // 5 bit signed long integer
4323 operand immL5()
4324 %{
4325 predicate(Assembler::is_simm(n->get_long(), 5));
4326 match(ConL);
4327
4328 op_cost(0);
4329 format %{ %}
4330 interface(CONST_INTER);
4331 %}
4332
4333 // 7 bit unsigned long integer
4334 operand immLU7()
4335 %{
4336 predicate(Assembler::is_uimm(n->get_long(), 7));
4337 match(ConL);
4338
4339 op_cost(0);
4340 format %{ %}
4341 interface(CONST_INTER);
4342 %}
4343
4344 // 8 bit signed value.
4345 operand immI8()
4346 %{
4347 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4348 match(ConI);
4349
4350 op_cost(0);
4351 format %{ %}
4352 interface(CONST_INTER);
4353 %}
4354
4355 // 8 bit signed value (simm8), or #simm8 LSL 8.
4356 operand immIDupV()
4357 %{
4358 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4359 match(ConI);
4360
4361 op_cost(0);
4362 format %{ %}
4363 interface(CONST_INTER);
4364 %}
4365
4366 // 8 bit signed value (simm8), or #simm8 LSL 8.
4367 operand immLDupV()
4368 %{
4369 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4370 match(ConL);
4371
4372 op_cost(0);
4373 format %{ %}
4374 interface(CONST_INTER);
4375 %}
4376
4377 // 8 bit signed value (simm8), or #simm8 LSL 8.
4378 operand immHDupV()
4379 %{
4380 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4381 match(ConH);
4382
4383 op_cost(0);
4384 format %{ %}
4385 interface(CONST_INTER);
4386 %}
4387
4388 // 8 bit integer valid for vector add sub immediate
4389 operand immBAddSubV()
4390 %{
4391 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4392 match(ConI);
4393
4394 op_cost(0);
4395 format %{ %}
4396 interface(CONST_INTER);
4397 %}
4398
4399 // 32 bit integer valid for add sub immediate
4400 operand immIAddSub()
4401 %{
4402 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4403 match(ConI);
4404 op_cost(0);
4405 format %{ %}
4406 interface(CONST_INTER);
4407 %}
4408
4409 // 32 bit integer valid for vector add sub immediate
4410 operand immIAddSubV()
4411 %{
4412 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4413 match(ConI);
4414
4415 op_cost(0);
4416 format %{ %}
4417 interface(CONST_INTER);
4418 %}
4419
4420 // 32 bit unsigned integer valid for logical immediate
4421
4422 operand immBLog()
4423 %{
4424 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4425 match(ConI);
4426
4427 op_cost(0);
4428 format %{ %}
4429 interface(CONST_INTER);
4430 %}
4431
4432 operand immSLog()
4433 %{
4434 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4435 match(ConI);
4436
4437 op_cost(0);
4438 format %{ %}
4439 interface(CONST_INTER);
4440 %}
4441
4442 operand immILog()
4443 %{
4444 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4445 match(ConI);
4446
4447 op_cost(0);
4448 format %{ %}
4449 interface(CONST_INTER);
4450 %}
4451
4452 // Integer operands 64 bit
4453 // 64 bit immediate
4454 operand immL()
4455 %{
4456 match(ConL);
4457
4458 op_cost(0);
4459 format %{ %}
4460 interface(CONST_INTER);
4461 %}
4462
4463 // 64 bit zero
4464 operand immL0()
4465 %{
4466 predicate(n->get_long() == 0);
4467 match(ConL);
4468
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // 64 bit unit decrement
4475 operand immL_M1()
4476 %{
4477 predicate(n->get_long() == -1);
4478 match(ConL);
4479
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 // 64 bit integer valid for add sub immediate
4486 operand immLAddSub()
4487 %{
4488 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4489 match(ConL);
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 // 64 bit integer valid for addv subv immediate
4496 operand immLAddSubV()
4497 %{
4498 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4499 match(ConL);
4500
4501 op_cost(0);
4502 format %{ %}
4503 interface(CONST_INTER);
4504 %}
4505
4506 // 64 bit integer valid for logical immediate
4507 operand immLLog()
4508 %{
4509 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4510 match(ConL);
4511 op_cost(0);
4512 format %{ %}
4513 interface(CONST_INTER);
4514 %}
4515
4516 // Long Immediate: low 32-bit mask
4517 operand immL_32bits()
4518 %{
4519 predicate(n->get_long() == 0xFFFFFFFFL);
4520 match(ConL);
4521 op_cost(0);
4522 format %{ %}
4523 interface(CONST_INTER);
4524 %}
4525
4526 // Pointer operands
4527 // Pointer Immediate
4528 operand immP()
4529 %{
4530 match(ConP);
4531
4532 op_cost(0);
4533 format %{ %}
4534 interface(CONST_INTER);
4535 %}
4536
4537 // nullptr Pointer Immediate
4538 operand immP0()
4539 %{
4540 predicate(n->get_ptr() == 0);
4541 match(ConP);
4542
4543 op_cost(0);
4544 format %{ %}
4545 interface(CONST_INTER);
4546 %}
4547
4548 // Pointer Immediate One
4549 // this is used in object initialization (initial object header)
4550 operand immP_1()
4551 %{
4552 predicate(n->get_ptr() == 1);
4553 match(ConP);
4554
4555 op_cost(0);
4556 format %{ %}
4557 interface(CONST_INTER);
4558 %}
4559
4560 // AOT Runtime Constants Address
4561 operand immAOTRuntimeConstantsAddress()
4562 %{
4563 // Check if the address is in the range of AOT Runtime Constants
4564 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4565 match(ConP);
4566
4567 op_cost(0);
4568 format %{ %}
4569 interface(CONST_INTER);
4570 %}
4571
4572 // Float and Double operands
4573 // Double Immediate
4574 operand immD()
4575 %{
4576 match(ConD);
4577 op_cost(0);
4578 format %{ %}
4579 interface(CONST_INTER);
4580 %}
4581
4582 // Double Immediate: +0.0d
4583 operand immD0()
4584 %{
4585 predicate(jlong_cast(n->getd()) == 0);
4586 match(ConD);
4587
4588 op_cost(0);
4589 format %{ %}
4590 interface(CONST_INTER);
4591 %}
4592
4593 // constant 'double +0.0'.
4594 operand immDPacked()
4595 %{
4596 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4597 match(ConD);
4598 op_cost(0);
4599 format %{ %}
4600 interface(CONST_INTER);
4601 %}
4602
4603 // Float Immediate
4604 operand immF()
4605 %{
4606 match(ConF);
4607 op_cost(0);
4608 format %{ %}
4609 interface(CONST_INTER);
4610 %}
4611
4612 // Float Immediate: +0.0f.
4613 operand immF0()
4614 %{
4615 predicate(jint_cast(n->getf()) == 0);
4616 match(ConF);
4617
4618 op_cost(0);
4619 format %{ %}
4620 interface(CONST_INTER);
4621 %}
4622
4623 // Half Float (FP16) Immediate
4624 operand immH()
4625 %{
4626 match(ConH);
4627 op_cost(0);
4628 format %{ %}
4629 interface(CONST_INTER);
4630 %}
4631
4632 //
4633 operand immFPacked()
4634 %{
4635 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4636 match(ConF);
4637 op_cost(0);
4638 format %{ %}
4639 interface(CONST_INTER);
4640 %}
4641
4642 // Narrow pointer operands
4643 // Narrow Pointer Immediate
4644 operand immN()
4645 %{
4646 match(ConN);
4647
4648 op_cost(0);
4649 format %{ %}
4650 interface(CONST_INTER);
4651 %}
4652
4653 // Narrow nullptr Pointer Immediate
4654 operand immN0()
4655 %{
4656 predicate(n->get_narrowcon() == 0);
4657 match(ConN);
4658
4659 op_cost(0);
4660 format %{ %}
4661 interface(CONST_INTER);
4662 %}
4663
4664 operand immNKlass()
4665 %{
4666 match(ConNKlass);
4667
4668 op_cost(0);
4669 format %{ %}
4670 interface(CONST_INTER);
4671 %}
4672
4673 // Integer 32 bit Register Operands
4674 // Integer 32 bitRegister (excludes SP)
4675 operand iRegI()
4676 %{
4677 constraint(ALLOC_IN_RC(any_reg32));
4678 match(RegI);
4679 match(iRegINoSp);
4680 op_cost(0);
4681 format %{ %}
4682 interface(REG_INTER);
4683 %}
4684
4685 // Integer 32 bit Register not Special
4686 operand iRegINoSp()
4687 %{
4688 constraint(ALLOC_IN_RC(no_special_reg32));
4689 match(RegI);
4690 op_cost(0);
4691 format %{ %}
4692 interface(REG_INTER);
4693 %}
4694
4695 // Integer 64 bit Register Operands
4696 // Integer 64 bit Register (includes SP)
4697 operand iRegL()
4698 %{
4699 constraint(ALLOC_IN_RC(any_reg));
4700 match(RegL);
4701 match(iRegLNoSp);
4702 op_cost(0);
4703 format %{ %}
4704 interface(REG_INTER);
4705 %}
4706
4707 // Integer 64 bit Register not Special
4708 operand iRegLNoSp()
4709 %{
4710 constraint(ALLOC_IN_RC(no_special_reg));
4711 match(RegL);
4712 match(iRegL_R0);
4713 format %{ %}
4714 interface(REG_INTER);
4715 %}
4716
4717 // Pointer Register Operands
4718 // Pointer Register
4719 operand iRegP()
4720 %{
4721 constraint(ALLOC_IN_RC(ptr_reg));
4722 match(RegP);
4723 match(iRegPNoSp);
4724 match(iRegP_R0);
4725 //match(iRegP_R2);
4726 //match(iRegP_R4);
4727 match(iRegP_R5);
4728 match(thread_RegP);
4729 op_cost(0);
4730 format %{ %}
4731 interface(REG_INTER);
4732 %}
4733
4734 // Pointer 64 bit Register not Special
4735 operand iRegPNoSp()
4736 %{
4737 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4738 match(RegP);
4739 // match(iRegP);
4740 // match(iRegP_R0);
4741 // match(iRegP_R2);
4742 // match(iRegP_R4);
4743 // match(iRegP_R5);
4744 // match(thread_RegP);
4745 op_cost(0);
4746 format %{ %}
4747 interface(REG_INTER);
4748 %}
4749
4750 // This operand is not allowed to use rfp even if
4751 // rfp is not used to hold the frame pointer.
4752 operand iRegPNoSpNoRfp()
4753 %{
4754 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4755 match(RegP);
4756 match(iRegPNoSp);
4757 op_cost(0);
4758 format %{ %}
4759 interface(REG_INTER);
4760 %}
4761
4762 // Pointer 64 bit Register R0 only
4763 operand iRegP_R0()
4764 %{
4765 constraint(ALLOC_IN_RC(r0_reg));
4766 match(RegP);
4767 // match(iRegP);
4768 match(iRegPNoSp);
4769 op_cost(0);
4770 format %{ %}
4771 interface(REG_INTER);
4772 %}
4773
4774 // Pointer 64 bit Register R1 only
4775 operand iRegP_R1()
4776 %{
4777 constraint(ALLOC_IN_RC(r1_reg));
4778 match(RegP);
4779 // match(iRegP);
4780 match(iRegPNoSp);
4781 op_cost(0);
4782 format %{ %}
4783 interface(REG_INTER);
4784 %}
4785
4786 // Pointer 64 bit Register R2 only
4787 operand iRegP_R2()
4788 %{
4789 constraint(ALLOC_IN_RC(r2_reg));
4790 match(RegP);
4791 // match(iRegP);
4792 match(iRegPNoSp);
4793 op_cost(0);
4794 format %{ %}
4795 interface(REG_INTER);
4796 %}
4797
4798 // Pointer 64 bit Register R3 only
4799 operand iRegP_R3()
4800 %{
4801 constraint(ALLOC_IN_RC(r3_reg));
4802 match(RegP);
4803 // match(iRegP);
4804 match(iRegPNoSp);
4805 op_cost(0);
4806 format %{ %}
4807 interface(REG_INTER);
4808 %}
4809
4810 // Pointer 64 bit Register R4 only
4811 operand iRegP_R4()
4812 %{
4813 constraint(ALLOC_IN_RC(r4_reg));
4814 match(RegP);
4815 // match(iRegP);
4816 match(iRegPNoSp);
4817 op_cost(0);
4818 format %{ %}
4819 interface(REG_INTER);
4820 %}
4821
4822 // Pointer 64 bit Register R5 only
4823 operand iRegP_R5()
4824 %{
4825 constraint(ALLOC_IN_RC(r5_reg));
4826 match(RegP);
4827 // match(iRegP);
4828 match(iRegPNoSp);
4829 op_cost(0);
4830 format %{ %}
4831 interface(REG_INTER);
4832 %}
4833
4834 // Pointer 64 bit Register R10 only
4835 operand iRegP_R10()
4836 %{
4837 constraint(ALLOC_IN_RC(r10_reg));
4838 match(RegP);
4839 // match(iRegP);
4840 match(iRegPNoSp);
4841 op_cost(0);
4842 format %{ %}
4843 interface(REG_INTER);
4844 %}
4845
4846 // Long 64 bit Register R0 only
4847 operand iRegL_R0()
4848 %{
4849 constraint(ALLOC_IN_RC(r0_reg));
4850 match(RegL);
4851 match(iRegLNoSp);
4852 op_cost(0);
4853 format %{ %}
4854 interface(REG_INTER);
4855 %}
4856
4857 // Long 64 bit Register R11 only
4858 operand iRegL_R11()
4859 %{
4860 constraint(ALLOC_IN_RC(r11_reg));
4861 match(RegL);
4862 match(iRegLNoSp);
4863 op_cost(0);
4864 format %{ %}
4865 interface(REG_INTER);
4866 %}
4867
4868 // Register R0 only
4869 operand iRegI_R0()
4870 %{
4871 constraint(ALLOC_IN_RC(int_r0_reg));
4872 match(RegI);
4873 match(iRegINoSp);
4874 op_cost(0);
4875 format %{ %}
4876 interface(REG_INTER);
4877 %}
4878
4879 // Register R2 only
4880 operand iRegI_R2()
4881 %{
4882 constraint(ALLOC_IN_RC(int_r2_reg));
4883 match(RegI);
4884 match(iRegINoSp);
4885 op_cost(0);
4886 format %{ %}
4887 interface(REG_INTER);
4888 %}
4889
4890 // Register R3 only
4891 operand iRegI_R3()
4892 %{
4893 constraint(ALLOC_IN_RC(int_r3_reg));
4894 match(RegI);
4895 match(iRegINoSp);
4896 op_cost(0);
4897 format %{ %}
4898 interface(REG_INTER);
4899 %}
4900
4901
4902 // Register R4 only
4903 operand iRegI_R4()
4904 %{
4905 constraint(ALLOC_IN_RC(int_r4_reg));
4906 match(RegI);
4907 match(iRegINoSp);
4908 op_cost(0);
4909 format %{ %}
4910 interface(REG_INTER);
4911 %}
4912
4913
4914 // Pointer Register Operands
4915 // Narrow Pointer Register
4916 operand iRegN()
4917 %{
4918 constraint(ALLOC_IN_RC(any_reg32));
4919 match(RegN);
4920 match(iRegNNoSp);
4921 op_cost(0);
4922 format %{ %}
4923 interface(REG_INTER);
4924 %}
4925
4926 // Integer 64 bit Register not Special
4927 operand iRegNNoSp()
4928 %{
4929 constraint(ALLOC_IN_RC(no_special_reg32));
4930 match(RegN);
4931 op_cost(0);
4932 format %{ %}
4933 interface(REG_INTER);
4934 %}
4935
4936 // Float Register
4937 // Float register operands
4938 operand vRegF()
4939 %{
4940 constraint(ALLOC_IN_RC(float_reg));
4941 match(RegF);
4942
4943 op_cost(0);
4944 format %{ %}
4945 interface(REG_INTER);
4946 %}
4947
4948 // Double Register
4949 // Double register operands
4950 operand vRegD()
4951 %{
4952 constraint(ALLOC_IN_RC(double_reg));
4953 match(RegD);
4954
4955 op_cost(0);
4956 format %{ %}
4957 interface(REG_INTER);
4958 %}
4959
4960 // Generic vector class. This will be used for
4961 // all vector operands, including NEON and SVE.
4962 operand vReg()
4963 %{
4964 constraint(ALLOC_IN_RC(dynamic));
4965 match(VecA);
4966 match(VecD);
4967 match(VecX);
4968
4969 op_cost(0);
4970 format %{ %}
4971 interface(REG_INTER);
4972 %}
4973
4974 operand vReg_V10()
4975 %{
4976 constraint(ALLOC_IN_RC(v10_veca_reg));
4977 match(vReg);
4978
4979 op_cost(0);
4980 format %{ %}
4981 interface(REG_INTER);
4982 %}
4983
4984 operand vReg_V11()
4985 %{
4986 constraint(ALLOC_IN_RC(v11_veca_reg));
4987 match(vReg);
4988
4989 op_cost(0);
4990 format %{ %}
4991 interface(REG_INTER);
4992 %}
4993
4994 operand vReg_V12()
4995 %{
4996 constraint(ALLOC_IN_RC(v12_veca_reg));
4997 match(vReg);
4998
4999 op_cost(0);
5000 format %{ %}
5001 interface(REG_INTER);
5002 %}
5003
5004 operand vReg_V13()
5005 %{
5006 constraint(ALLOC_IN_RC(v13_veca_reg));
5007 match(vReg);
5008
5009 op_cost(0);
5010 format %{ %}
5011 interface(REG_INTER);
5012 %}
5013
5014 operand vReg_V17()
5015 %{
5016 constraint(ALLOC_IN_RC(v17_veca_reg));
5017 match(vReg);
5018
5019 op_cost(0);
5020 format %{ %}
5021 interface(REG_INTER);
5022 %}
5023
5024 operand vReg_V18()
5025 %{
5026 constraint(ALLOC_IN_RC(v18_veca_reg));
5027 match(vReg);
5028
5029 op_cost(0);
5030 format %{ %}
5031 interface(REG_INTER);
5032 %}
5033
5034 operand vReg_V23()
5035 %{
5036 constraint(ALLOC_IN_RC(v23_veca_reg));
5037 match(vReg);
5038
5039 op_cost(0);
5040 format %{ %}
5041 interface(REG_INTER);
5042 %}
5043
5044 operand vReg_V24()
5045 %{
5046 constraint(ALLOC_IN_RC(v24_veca_reg));
5047 match(vReg);
5048
5049 op_cost(0);
5050 format %{ %}
5051 interface(REG_INTER);
5052 %}
5053
5054 operand vecA()
5055 %{
5056 constraint(ALLOC_IN_RC(vectora_reg));
5057 match(VecA);
5058
5059 op_cost(0);
5060 format %{ %}
5061 interface(REG_INTER);
5062 %}
5063
5064 operand vecD()
5065 %{
5066 constraint(ALLOC_IN_RC(vectord_reg));
5067 match(VecD);
5068
5069 op_cost(0);
5070 format %{ %}
5071 interface(REG_INTER);
5072 %}
5073
5074 operand vecX()
5075 %{
5076 constraint(ALLOC_IN_RC(vectorx_reg));
5077 match(VecX);
5078
5079 op_cost(0);
5080 format %{ %}
5081 interface(REG_INTER);
5082 %}
5083
5084 operand vRegD_V0()
5085 %{
5086 constraint(ALLOC_IN_RC(v0_reg));
5087 match(RegD);
5088 op_cost(0);
5089 format %{ %}
5090 interface(REG_INTER);
5091 %}
5092
5093 operand vRegD_V1()
5094 %{
5095 constraint(ALLOC_IN_RC(v1_reg));
5096 match(RegD);
5097 op_cost(0);
5098 format %{ %}
5099 interface(REG_INTER);
5100 %}
5101
5102 operand vRegD_V2()
5103 %{
5104 constraint(ALLOC_IN_RC(v2_reg));
5105 match(RegD);
5106 op_cost(0);
5107 format %{ %}
5108 interface(REG_INTER);
5109 %}
5110
5111 operand vRegD_V3()
5112 %{
5113 constraint(ALLOC_IN_RC(v3_reg));
5114 match(RegD);
5115 op_cost(0);
5116 format %{ %}
5117 interface(REG_INTER);
5118 %}
5119
5120 operand vRegD_V4()
5121 %{
5122 constraint(ALLOC_IN_RC(v4_reg));
5123 match(RegD);
5124 op_cost(0);
5125 format %{ %}
5126 interface(REG_INTER);
5127 %}
5128
5129 operand vRegD_V5()
5130 %{
5131 constraint(ALLOC_IN_RC(v5_reg));
5132 match(RegD);
5133 op_cost(0);
5134 format %{ %}
5135 interface(REG_INTER);
5136 %}
5137
5138 operand vRegD_V6()
5139 %{
5140 constraint(ALLOC_IN_RC(v6_reg));
5141 match(RegD);
5142 op_cost(0);
5143 format %{ %}
5144 interface(REG_INTER);
5145 %}
5146
5147 operand vRegD_V7()
5148 %{
5149 constraint(ALLOC_IN_RC(v7_reg));
5150 match(RegD);
5151 op_cost(0);
5152 format %{ %}
5153 interface(REG_INTER);
5154 %}
5155
5156 operand vRegD_V12()
5157 %{
5158 constraint(ALLOC_IN_RC(v12_reg));
5159 match(RegD);
5160 op_cost(0);
5161 format %{ %}
5162 interface(REG_INTER);
5163 %}
5164
5165 operand vRegD_V13()
5166 %{
5167 constraint(ALLOC_IN_RC(v13_reg));
5168 match(RegD);
5169 op_cost(0);
5170 format %{ %}
5171 interface(REG_INTER);
5172 %}
5173
5174 operand pReg()
5175 %{
5176 constraint(ALLOC_IN_RC(pr_reg));
5177 match(RegVectMask);
5178 match(pRegGov);
5179 op_cost(0);
5180 format %{ %}
5181 interface(REG_INTER);
5182 %}
5183
5184 operand pRegGov()
5185 %{
5186 constraint(ALLOC_IN_RC(gov_pr));
5187 match(RegVectMask);
5188 match(pReg);
5189 op_cost(0);
5190 format %{ %}
5191 interface(REG_INTER);
5192 %}
5193
5194 operand pRegGov_P0()
5195 %{
5196 constraint(ALLOC_IN_RC(p0_reg));
5197 match(RegVectMask);
5198 op_cost(0);
5199 format %{ %}
5200 interface(REG_INTER);
5201 %}
5202
5203 operand pRegGov_P1()
5204 %{
5205 constraint(ALLOC_IN_RC(p1_reg));
5206 match(RegVectMask);
5207 op_cost(0);
5208 format %{ %}
5209 interface(REG_INTER);
5210 %}
5211
5212 // Flags register, used as output of signed compare instructions
5213
5214 // note that on AArch64 we also use this register as the output for
5215 // for floating point compare instructions (CmpF CmpD). this ensures
5216 // that ordered inequality tests use GT, GE, LT or LE none of which
5217 // pass through cases where the result is unordered i.e. one or both
5218 // inputs to the compare is a NaN. this means that the ideal code can
5219 // replace e.g. a GT with an LE and not end up capturing the NaN case
5220 // (where the comparison should always fail). EQ and NE tests are
5221 // always generated in ideal code so that unordered folds into the NE
5222 // case, matching the behaviour of AArch64 NE.
5223 //
5224 // This differs from x86 where the outputs of FP compares use a
5225 // special FP flags registers and where compares based on this
5226 // register are distinguished into ordered inequalities (cmpOpUCF) and
5227 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5228 // to explicitly handle the unordered case in branches. x86 also has
5229 // to include extra CMoveX rules to accept a cmpOpUCF input.
5230
5231 operand rFlagsReg()
5232 %{
5233 constraint(ALLOC_IN_RC(int_flags));
5234 match(RegFlags);
5235
5236 op_cost(0);
5237 format %{ "RFLAGS" %}
5238 interface(REG_INTER);
5239 %}
5240
5241 // Flags register, used as output of unsigned compare instructions
5242 operand rFlagsRegU()
5243 %{
5244 constraint(ALLOC_IN_RC(int_flags));
5245 match(RegFlags);
5246
5247 op_cost(0);
5248 format %{ "RFLAGSU" %}
5249 interface(REG_INTER);
5250 %}
5251
5252 // Special Registers
5253
5254 // Method Register
5255 operand inline_cache_RegP(iRegP reg)
5256 %{
5257 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5258 match(reg);
5259 match(iRegPNoSp);
5260 op_cost(0);
5261 format %{ %}
5262 interface(REG_INTER);
5263 %}
5264
5265 // Thread Register
5266 operand thread_RegP(iRegP reg)
5267 %{
5268 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5269 match(reg);
5270 op_cost(0);
5271 format %{ %}
5272 interface(REG_INTER);
5273 %}
5274
5275 //----------Memory Operands----------------------------------------------------
5276
5277 operand indirect(iRegP reg)
5278 %{
5279 constraint(ALLOC_IN_RC(ptr_reg));
5280 match(reg);
5281 op_cost(0);
5282 format %{ "[$reg]" %}
5283 interface(MEMORY_INTER) %{
5284 base($reg);
5285 index(0xffffffff);
5286 scale(0x0);
5287 disp(0x0);
5288 %}
5289 %}
5290
5291 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5292 %{
5293 constraint(ALLOC_IN_RC(ptr_reg));
5294 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5295 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5296 op_cost(0);
5297 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5298 interface(MEMORY_INTER) %{
5299 base($reg);
5300 index($ireg);
5301 scale($scale);
5302 disp(0x0);
5303 %}
5304 %}
5305
5306 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5307 %{
5308 constraint(ALLOC_IN_RC(ptr_reg));
5309 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5310 match(AddP reg (LShiftL lreg scale));
5311 op_cost(0);
5312 format %{ "$reg, $lreg lsl($scale)" %}
5313 interface(MEMORY_INTER) %{
5314 base($reg);
5315 index($lreg);
5316 scale($scale);
5317 disp(0x0);
5318 %}
5319 %}
5320
5321 operand indIndexI2L(iRegP reg, iRegI ireg)
5322 %{
5323 constraint(ALLOC_IN_RC(ptr_reg));
5324 match(AddP reg (ConvI2L ireg));
5325 op_cost(0);
5326 format %{ "$reg, $ireg, 0, I2L" %}
5327 interface(MEMORY_INTER) %{
5328 base($reg);
5329 index($ireg);
5330 scale(0x0);
5331 disp(0x0);
5332 %}
5333 %}
5334
5335 operand indIndex(iRegP reg, iRegL lreg)
5336 %{
5337 constraint(ALLOC_IN_RC(ptr_reg));
5338 match(AddP reg lreg);
5339 op_cost(0);
5340 format %{ "$reg, $lreg" %}
5341 interface(MEMORY_INTER) %{
5342 base($reg);
5343 index($lreg);
5344 scale(0x0);
5345 disp(0x0);
5346 %}
5347 %}
5348
5349 operand indOffI1(iRegP reg, immIOffset1 off)
5350 %{
5351 constraint(ALLOC_IN_RC(ptr_reg));
5352 match(AddP reg off);
5353 op_cost(0);
5354 format %{ "[$reg, $off]" %}
5355 interface(MEMORY_INTER) %{
5356 base($reg);
5357 index(0xffffffff);
5358 scale(0x0);
5359 disp($off);
5360 %}
5361 %}
5362
5363 operand indOffI2(iRegP reg, immIOffset2 off)
5364 %{
5365 constraint(ALLOC_IN_RC(ptr_reg));
5366 match(AddP reg off);
5367 op_cost(0);
5368 format %{ "[$reg, $off]" %}
5369 interface(MEMORY_INTER) %{
5370 base($reg);
5371 index(0xffffffff);
5372 scale(0x0);
5373 disp($off);
5374 %}
5375 %}
5376
5377 operand indOffI4(iRegP reg, immIOffset4 off)
5378 %{
5379 constraint(ALLOC_IN_RC(ptr_reg));
5380 match(AddP reg off);
5381 op_cost(0);
5382 format %{ "[$reg, $off]" %}
5383 interface(MEMORY_INTER) %{
5384 base($reg);
5385 index(0xffffffff);
5386 scale(0x0);
5387 disp($off);
5388 %}
5389 %}
5390
5391 operand indOffI8(iRegP reg, immIOffset8 off)
5392 %{
5393 constraint(ALLOC_IN_RC(ptr_reg));
5394 match(AddP reg off);
5395 op_cost(0);
5396 format %{ "[$reg, $off]" %}
5397 interface(MEMORY_INTER) %{
5398 base($reg);
5399 index(0xffffffff);
5400 scale(0x0);
5401 disp($off);
5402 %}
5403 %}
5404
5405 operand indOffI16(iRegP reg, immIOffset16 off)
5406 %{
5407 constraint(ALLOC_IN_RC(ptr_reg));
5408 match(AddP reg off);
5409 op_cost(0);
5410 format %{ "[$reg, $off]" %}
5411 interface(MEMORY_INTER) %{
5412 base($reg);
5413 index(0xffffffff);
5414 scale(0x0);
5415 disp($off);
5416 %}
5417 %}
5418
5419 operand indOffL1(iRegP reg, immLoffset1 off)
5420 %{
5421 constraint(ALLOC_IN_RC(ptr_reg));
5422 match(AddP reg off);
5423 op_cost(0);
5424 format %{ "[$reg, $off]" %}
5425 interface(MEMORY_INTER) %{
5426 base($reg);
5427 index(0xffffffff);
5428 scale(0x0);
5429 disp($off);
5430 %}
5431 %}
5432
5433 operand indOffL2(iRegP reg, immLoffset2 off)
5434 %{
5435 constraint(ALLOC_IN_RC(ptr_reg));
5436 match(AddP reg off);
5437 op_cost(0);
5438 format %{ "[$reg, $off]" %}
5439 interface(MEMORY_INTER) %{
5440 base($reg);
5441 index(0xffffffff);
5442 scale(0x0);
5443 disp($off);
5444 %}
5445 %}
5446
5447 operand indOffL4(iRegP reg, immLoffset4 off)
5448 %{
5449 constraint(ALLOC_IN_RC(ptr_reg));
5450 match(AddP reg off);
5451 op_cost(0);
5452 format %{ "[$reg, $off]" %}
5453 interface(MEMORY_INTER) %{
5454 base($reg);
5455 index(0xffffffff);
5456 scale(0x0);
5457 disp($off);
5458 %}
5459 %}
5460
5461 operand indOffL8(iRegP reg, immLoffset8 off)
5462 %{
5463 constraint(ALLOC_IN_RC(ptr_reg));
5464 match(AddP reg off);
5465 op_cost(0);
5466 format %{ "[$reg, $off]" %}
5467 interface(MEMORY_INTER) %{
5468 base($reg);
5469 index(0xffffffff);
5470 scale(0x0);
5471 disp($off);
5472 %}
5473 %}
5474
5475 operand indOffL16(iRegP reg, immLoffset16 off)
5476 %{
5477 constraint(ALLOC_IN_RC(ptr_reg));
5478 match(AddP reg off);
5479 op_cost(0);
5480 format %{ "[$reg, $off]" %}
5481 interface(MEMORY_INTER) %{
5482 base($reg);
5483 index(0xffffffff);
5484 scale(0x0);
5485 disp($off);
5486 %}
5487 %}
5488
5489 operand indirectX2P(iRegL reg)
5490 %{
5491 constraint(ALLOC_IN_RC(ptr_reg));
5492 match(CastX2P reg);
5493 op_cost(0);
5494 format %{ "[$reg]\t# long -> ptr" %}
5495 interface(MEMORY_INTER) %{
5496 base($reg);
5497 index(0xffffffff);
5498 scale(0x0);
5499 disp(0x0);
5500 %}
5501 %}
5502
5503 operand indOffX2P(iRegL reg, immLOffset off)
5504 %{
5505 constraint(ALLOC_IN_RC(ptr_reg));
5506 match(AddP (CastX2P reg) off);
5507 op_cost(0);
5508 format %{ "[$reg, $off]\t# long -> ptr" %}
5509 interface(MEMORY_INTER) %{
5510 base($reg);
5511 index(0xffffffff);
5512 scale(0x0);
5513 disp($off);
5514 %}
5515 %}
5516
5517 operand indirectN(iRegN reg)
5518 %{
5519 predicate(CompressedOops::shift() == 0);
5520 constraint(ALLOC_IN_RC(ptr_reg));
5521 match(DecodeN reg);
5522 op_cost(0);
5523 format %{ "[$reg]\t# narrow" %}
5524 interface(MEMORY_INTER) %{
5525 base($reg);
5526 index(0xffffffff);
5527 scale(0x0);
5528 disp(0x0);
5529 %}
5530 %}
5531
5532 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5533 %{
5534 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5535 constraint(ALLOC_IN_RC(ptr_reg));
5536 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5537 op_cost(0);
5538 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5539 interface(MEMORY_INTER) %{
5540 base($reg);
5541 index($ireg);
5542 scale($scale);
5543 disp(0x0);
5544 %}
5545 %}
5546
5547 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5548 %{
5549 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5550 constraint(ALLOC_IN_RC(ptr_reg));
5551 match(AddP (DecodeN reg) (LShiftL lreg scale));
5552 op_cost(0);
5553 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5554 interface(MEMORY_INTER) %{
5555 base($reg);
5556 index($lreg);
5557 scale($scale);
5558 disp(0x0);
5559 %}
5560 %}
5561
5562 operand indIndexI2LN(iRegN reg, iRegI ireg)
5563 %{
5564 predicate(CompressedOops::shift() == 0);
5565 constraint(ALLOC_IN_RC(ptr_reg));
5566 match(AddP (DecodeN reg) (ConvI2L ireg));
5567 op_cost(0);
5568 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5569 interface(MEMORY_INTER) %{
5570 base($reg);
5571 index($ireg);
5572 scale(0x0);
5573 disp(0x0);
5574 %}
5575 %}
5576
5577 operand indIndexN(iRegN reg, iRegL lreg)
5578 %{
5579 predicate(CompressedOops::shift() == 0);
5580 constraint(ALLOC_IN_RC(ptr_reg));
5581 match(AddP (DecodeN reg) lreg);
5582 op_cost(0);
5583 format %{ "$reg, $lreg\t# narrow" %}
5584 interface(MEMORY_INTER) %{
5585 base($reg);
5586 index($lreg);
5587 scale(0x0);
5588 disp(0x0);
5589 %}
5590 %}
5591
5592 operand indOffIN(iRegN reg, immIOffset off)
5593 %{
5594 predicate(CompressedOops::shift() == 0);
5595 constraint(ALLOC_IN_RC(ptr_reg));
5596 match(AddP (DecodeN reg) off);
5597 op_cost(0);
5598 format %{ "[$reg, $off]\t# narrow" %}
5599 interface(MEMORY_INTER) %{
5600 base($reg);
5601 index(0xffffffff);
5602 scale(0x0);
5603 disp($off);
5604 %}
5605 %}
5606
5607 operand indOffLN(iRegN reg, immLOffset off)
5608 %{
5609 predicate(CompressedOops::shift() == 0);
5610 constraint(ALLOC_IN_RC(ptr_reg));
5611 match(AddP (DecodeN reg) off);
5612 op_cost(0);
5613 format %{ "[$reg, $off]\t# narrow" %}
5614 interface(MEMORY_INTER) %{
5615 base($reg);
5616 index(0xffffffff);
5617 scale(0x0);
5618 disp($off);
5619 %}
5620 %}
5621
5622
5623 //----------Special Memory Operands--------------------------------------------
5624 // Stack Slot Operand - This operand is used for loading and storing temporary
5625 // values on the stack where a match requires a value to
5626 // flow through memory.
5627 operand stackSlotP(sRegP reg)
5628 %{
5629 constraint(ALLOC_IN_RC(stack_slots));
5630 op_cost(100);
5631 // No match rule because this operand is only generated in matching
5632 // match(RegP);
5633 format %{ "[$reg]" %}
5634 interface(MEMORY_INTER) %{
5635 base(0x1e); // RSP
5636 index(0x0); // No Index
5637 scale(0x0); // No Scale
5638 disp($reg); // Stack Offset
5639 %}
5640 %}
5641
5642 operand stackSlotI(sRegI reg)
5643 %{
5644 constraint(ALLOC_IN_RC(stack_slots));
5645 // No match rule because this operand is only generated in matching
5646 // match(RegI);
5647 format %{ "[$reg]" %}
5648 interface(MEMORY_INTER) %{
5649 base(0x1e); // RSP
5650 index(0x0); // No Index
5651 scale(0x0); // No Scale
5652 disp($reg); // Stack Offset
5653 %}
5654 %}
5655
5656 operand stackSlotF(sRegF reg)
5657 %{
5658 constraint(ALLOC_IN_RC(stack_slots));
5659 // No match rule because this operand is only generated in matching
5660 // match(RegF);
5661 format %{ "[$reg]" %}
5662 interface(MEMORY_INTER) %{
5663 base(0x1e); // RSP
5664 index(0x0); // No Index
5665 scale(0x0); // No Scale
5666 disp($reg); // Stack Offset
5667 %}
5668 %}
5669
5670 operand stackSlotD(sRegD reg)
5671 %{
5672 constraint(ALLOC_IN_RC(stack_slots));
5673 // No match rule because this operand is only generated in matching
5674 // match(RegD);
5675 format %{ "[$reg]" %}
5676 interface(MEMORY_INTER) %{
5677 base(0x1e); // RSP
5678 index(0x0); // No Index
5679 scale(0x0); // No Scale
5680 disp($reg); // Stack Offset
5681 %}
5682 %}
5683
5684 operand stackSlotL(sRegL reg)
5685 %{
5686 constraint(ALLOC_IN_RC(stack_slots));
5687 // No match rule because this operand is only generated in matching
5688 // match(RegL);
5689 format %{ "[$reg]" %}
5690 interface(MEMORY_INTER) %{
5691 base(0x1e); // RSP
5692 index(0x0); // No Index
5693 scale(0x0); // No Scale
5694 disp($reg); // Stack Offset
5695 %}
5696 %}
5697
5698 // Operands for expressing Control Flow
5699 // NOTE: Label is a predefined operand which should not be redefined in
5700 // the AD file. It is generically handled within the ADLC.
5701
5702 //----------Conditional Branch Operands----------------------------------------
5703 // Comparison Op - This is the operation of the comparison, and is limited to
5704 // the following set of codes:
5705 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5706 //
5707 // Other attributes of the comparison, such as unsignedness, are specified
5708 // by the comparison instruction that sets a condition code flags register.
5709 // That result is represented by a flags operand whose subtype is appropriate
5710 // to the unsignedness (etc.) of the comparison.
5711 //
5712 // Later, the instruction which matches both the Comparison Op (a Bool) and
5713 // the flags (produced by the Cmp) specifies the coding of the comparison op
5714 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5715
5716 // used for signed integral comparisons and fp comparisons
5717
5718 operand cmpOp()
5719 %{
5720 match(Bool);
5721
5722 format %{ "" %}
5723 interface(COND_INTER) %{
5724 equal(0x0, "eq");
5725 not_equal(0x1, "ne");
5726 less(0xb, "lt");
5727 greater_equal(0xa, "ge");
5728 less_equal(0xd, "le");
5729 greater(0xc, "gt");
5730 overflow(0x6, "vs");
5731 no_overflow(0x7, "vc");
5732 %}
5733 %}
5734
5735 // used for unsigned integral comparisons
5736
5737 operand cmpOpU()
5738 %{
5739 match(Bool);
5740
5741 format %{ "" %}
5742 interface(COND_INTER) %{
5743 equal(0x0, "eq");
5744 not_equal(0x1, "ne");
5745 less(0x3, "lo");
5746 greater_equal(0x2, "hs");
5747 less_equal(0x9, "ls");
5748 greater(0x8, "hi");
5749 overflow(0x6, "vs");
5750 no_overflow(0x7, "vc");
5751 %}
5752 %}
5753
5754 // used for certain integral comparisons which can be
5755 // converted to cbxx or tbxx instructions
5756
5757 operand cmpOpEqNe()
5758 %{
5759 match(Bool);
5760 op_cost(0);
5761 predicate(n->as_Bool()->_test._test == BoolTest::ne
5762 || n->as_Bool()->_test._test == BoolTest::eq);
5763
5764 format %{ "" %}
5765 interface(COND_INTER) %{
5766 equal(0x0, "eq");
5767 not_equal(0x1, "ne");
5768 less(0xb, "lt");
5769 greater_equal(0xa, "ge");
5770 less_equal(0xd, "le");
5771 greater(0xc, "gt");
5772 overflow(0x6, "vs");
5773 no_overflow(0x7, "vc");
5774 %}
5775 %}
5776
5777 // used for certain integral comparisons which can be
5778 // converted to cbxx or tbxx instructions
5779
5780 operand cmpOpLtGe()
5781 %{
5782 match(Bool);
5783 op_cost(0);
5784
5785 predicate(n->as_Bool()->_test._test == BoolTest::lt
5786 || n->as_Bool()->_test._test == BoolTest::ge);
5787
5788 format %{ "" %}
5789 interface(COND_INTER) %{
5790 equal(0x0, "eq");
5791 not_equal(0x1, "ne");
5792 less(0xb, "lt");
5793 greater_equal(0xa, "ge");
5794 less_equal(0xd, "le");
5795 greater(0xc, "gt");
5796 overflow(0x6, "vs");
5797 no_overflow(0x7, "vc");
5798 %}
5799 %}
5800
5801 // used for certain unsigned integral comparisons which can be
5802 // converted to cbxx or tbxx instructions
5803
5804 operand cmpOpUEqNeLeGt()
5805 %{
5806 match(Bool);
5807 op_cost(0);
5808
5809 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5810 n->as_Bool()->_test._test == BoolTest::ne ||
5811 n->as_Bool()->_test._test == BoolTest::le ||
5812 n->as_Bool()->_test._test == BoolTest::gt);
5813
5814 format %{ "" %}
5815 interface(COND_INTER) %{
5816 equal(0x0, "eq");
5817 not_equal(0x1, "ne");
5818 less(0x3, "lo");
5819 greater_equal(0x2, "hs");
5820 less_equal(0x9, "ls");
5821 greater(0x8, "hi");
5822 overflow(0x6, "vs");
5823 no_overflow(0x7, "vc");
5824 %}
5825 %}
5826
5827 // Special operand allowing long args to int ops to be truncated for free
5828
5829 operand iRegL2I(iRegL reg) %{
5830
5831 op_cost(0);
5832
5833 match(ConvL2I reg);
5834
5835 format %{ "l2i($reg)" %}
5836
5837 interface(REG_INTER)
5838 %}
5839
5840 operand iRegL2P(iRegL reg) %{
5841
5842 op_cost(0);
5843
5844 match(CastX2P reg);
5845
5846 format %{ "l2p($reg)" %}
5847
5848 interface(REG_INTER)
5849 %}
5850
5851 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5852 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5853 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5854 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5855
5856 //----------OPERAND CLASSES----------------------------------------------------
5857 // Operand Classes are groups of operands that are used as to simplify
5858 // instruction definitions by not requiring the AD writer to specify
5859 // separate instructions for every form of operand when the
5860 // instruction accepts multiple operand types with the same basic
5861 // encoding and format. The classic case of this is memory operands.
5862
5863 // memory is used to define read/write location for load/store
5864 // instruction defs. we can turn a memory op into an Address
5865
5866 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5867 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5868
5869 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5870 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5871
5872 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5873 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5874
5875 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5876 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5877
5878 // All of the memory operands. For the pipeline description.
5879 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5880 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5881 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5882
5883
5884 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5885 // operations. it allows the src to be either an iRegI or a (ConvL2I
5886 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5887 // can be elided because the 32-bit instruction will just employ the
5888 // lower 32 bits anyway.
5889 //
5890 // n.b. this does not elide all L2I conversions. if the truncated
5891 // value is consumed by more than one operation then the ConvL2I
5892 // cannot be bundled into the consuming nodes so an l2i gets planted
5893 // (actually a movw $dst $src) and the downstream instructions consume
5894 // the result of the l2i as an iRegI input. That's a shame since the
5895 // movw is actually redundant but its not too costly.
5896
5897 opclass iRegIorL2I(iRegI, iRegL2I);
5898 opclass iRegPorL2P(iRegP, iRegL2P);
5899
5900 //----------PIPELINE-----------------------------------------------------------
5901 // Rules which define the behavior of the target architectures pipeline.
5902
5903 // For specific pipelines, eg A53, define the stages of that pipeline
5904 //pipe_desc(ISS, EX1, EX2, WR);
5905 #define ISS S0
5906 #define EX1 S1
5907 #define EX2 S2
5908 #define WR S3
5909
5910 // Integer ALU reg operation
5911 pipeline %{
5912
5913 attributes %{
5914 // ARM instructions are of fixed length
5915 fixed_size_instructions; // Fixed size instructions TODO does
5916 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5917 // ARM instructions come in 32-bit word units
5918 instruction_unit_size = 4; // An instruction is 4 bytes long
5919 instruction_fetch_unit_size = 64; // The processor fetches one line
5920 instruction_fetch_units = 1; // of 64 bytes
5921 %}
5922
5923 // We don't use an actual pipeline model so don't care about resources
5924 // or description. we do use pipeline classes to introduce fixed
5925 // latencies
5926
5927 //----------RESOURCES----------------------------------------------------------
5928 // Resources are the functional units available to the machine
5929
5930 resources( INS0, INS1, INS01 = INS0 | INS1,
5931 ALU0, ALU1, ALU = ALU0 | ALU1,
5932 MAC,
5933 DIV,
5934 BRANCH,
5935 LDST,
5936 NEON_FP);
5937
5938 //----------PIPELINE DESCRIPTION-----------------------------------------------
5939 // Pipeline Description specifies the stages in the machine's pipeline
5940
5941 // Define the pipeline as a generic 6 stage pipeline
5942 pipe_desc(S0, S1, S2, S3, S4, S5);
5943
5944 //----------PIPELINE CLASSES---------------------------------------------------
5945 // Pipeline Classes describe the stages in which input and output are
5946 // referenced by the hardware pipeline.
5947
5948 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5949 %{
5950 single_instruction;
5951 src1 : S1(read);
5952 src2 : S2(read);
5953 dst : S5(write);
5954 INS01 : ISS;
5955 NEON_FP : S5;
5956 %}
5957
5958 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5959 %{
5960 single_instruction;
5961 src1 : S1(read);
5962 src2 : S2(read);
5963 dst : S5(write);
5964 INS01 : ISS;
5965 NEON_FP : S5;
5966 %}
5967
5968 pipe_class fp_uop_s(vRegF dst, vRegF src)
5969 %{
5970 single_instruction;
5971 src : S1(read);
5972 dst : S5(write);
5973 INS01 : ISS;
5974 NEON_FP : S5;
5975 %}
5976
5977 pipe_class fp_uop_d(vRegD dst, vRegD src)
5978 %{
5979 single_instruction;
5980 src : S1(read);
5981 dst : S5(write);
5982 INS01 : ISS;
5983 NEON_FP : S5;
5984 %}
5985
5986 pipe_class fp_d2f(vRegF dst, vRegD src)
5987 %{
5988 single_instruction;
5989 src : S1(read);
5990 dst : S5(write);
5991 INS01 : ISS;
5992 NEON_FP : S5;
5993 %}
5994
5995 pipe_class fp_f2d(vRegD dst, vRegF src)
5996 %{
5997 single_instruction;
5998 src : S1(read);
5999 dst : S5(write);
6000 INS01 : ISS;
6001 NEON_FP : S5;
6002 %}
6003
6004 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6005 %{
6006 single_instruction;
6007 src : S1(read);
6008 dst : S5(write);
6009 INS01 : ISS;
6010 NEON_FP : S5;
6011 %}
6012
6013 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6014 %{
6015 single_instruction;
6016 src : S1(read);
6017 dst : S5(write);
6018 INS01 : ISS;
6019 NEON_FP : S5;
6020 %}
6021
6022 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6023 %{
6024 single_instruction;
6025 src : S1(read);
6026 dst : S5(write);
6027 INS01 : ISS;
6028 NEON_FP : S5;
6029 %}
6030
6031 pipe_class fp_l2f(vRegF dst, iRegL src)
6032 %{
6033 single_instruction;
6034 src : S1(read);
6035 dst : S5(write);
6036 INS01 : ISS;
6037 NEON_FP : S5;
6038 %}
6039
6040 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6041 %{
6042 single_instruction;
6043 src : S1(read);
6044 dst : S5(write);
6045 INS01 : ISS;
6046 NEON_FP : S5;
6047 %}
6048
6049 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6050 %{
6051 single_instruction;
6052 src : S1(read);
6053 dst : S5(write);
6054 INS01 : ISS;
6055 NEON_FP : S5;
6056 %}
6057
6058 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6059 %{
6060 single_instruction;
6061 src : S1(read);
6062 dst : S5(write);
6063 INS01 : ISS;
6064 NEON_FP : S5;
6065 %}
6066
6067 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6068 %{
6069 single_instruction;
6070 src : S1(read);
6071 dst : S5(write);
6072 INS01 : ISS;
6073 NEON_FP : S5;
6074 %}
6075
6076 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6077 %{
6078 single_instruction;
6079 src1 : S1(read);
6080 src2 : S2(read);
6081 dst : S5(write);
6082 INS0 : ISS;
6083 NEON_FP : S5;
6084 %}
6085
6086 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6087 %{
6088 single_instruction;
6089 src1 : S1(read);
6090 src2 : S2(read);
6091 dst : S5(write);
6092 INS0 : ISS;
6093 NEON_FP : S5;
6094 %}
6095
6096 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6097 %{
6098 single_instruction;
6099 cr : S1(read);
6100 src1 : S1(read);
6101 src2 : S1(read);
6102 dst : S3(write);
6103 INS01 : ISS;
6104 NEON_FP : S3;
6105 %}
6106
6107 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6108 %{
6109 single_instruction;
6110 cr : S1(read);
6111 src1 : S1(read);
6112 src2 : S1(read);
6113 dst : S3(write);
6114 INS01 : ISS;
6115 NEON_FP : S3;
6116 %}
6117
6118 pipe_class fp_imm_s(vRegF dst)
6119 %{
6120 single_instruction;
6121 dst : S3(write);
6122 INS01 : ISS;
6123 NEON_FP : S3;
6124 %}
6125
6126 pipe_class fp_imm_d(vRegD dst)
6127 %{
6128 single_instruction;
6129 dst : S3(write);
6130 INS01 : ISS;
6131 NEON_FP : S3;
6132 %}
6133
6134 pipe_class fp_load_constant_s(vRegF dst)
6135 %{
6136 single_instruction;
6137 dst : S4(write);
6138 INS01 : ISS;
6139 NEON_FP : S4;
6140 %}
6141
6142 pipe_class fp_load_constant_d(vRegD dst)
6143 %{
6144 single_instruction;
6145 dst : S4(write);
6146 INS01 : ISS;
6147 NEON_FP : S4;
6148 %}
6149
6150 //------- Integer ALU operations --------------------------
6151
6152 // Integer ALU reg-reg operation
6153 // Operands needed in EX1, result generated in EX2
6154 // Eg. ADD x0, x1, x2
6155 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6156 %{
6157 single_instruction;
6158 dst : EX2(write);
6159 src1 : EX1(read);
6160 src2 : EX1(read);
6161 INS01 : ISS; // Dual issue as instruction 0 or 1
6162 ALU : EX2;
6163 %}
6164
6165 // Integer ALU reg-reg operation with constant shift
6166 // Shifted register must be available in LATE_ISS instead of EX1
6167 // Eg. ADD x0, x1, x2, LSL #2
6168 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6169 %{
6170 single_instruction;
6171 dst : EX2(write);
6172 src1 : EX1(read);
6173 src2 : ISS(read);
6174 INS01 : ISS;
6175 ALU : EX2;
6176 %}
6177
6178 // Integer ALU reg operation with constant shift
6179 // Eg. LSL x0, x1, #shift
6180 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6181 %{
6182 single_instruction;
6183 dst : EX2(write);
6184 src1 : ISS(read);
6185 INS01 : ISS;
6186 ALU : EX2;
6187 %}
6188
6189 // Integer ALU reg-reg operation with variable shift
6190 // Both operands must be available in LATE_ISS instead of EX1
6191 // Result is available in EX1 instead of EX2
6192 // Eg. LSLV x0, x1, x2
6193 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6194 %{
6195 single_instruction;
6196 dst : EX1(write);
6197 src1 : ISS(read);
6198 src2 : ISS(read);
6199 INS01 : ISS;
6200 ALU : EX1;
6201 %}
6202
6203 // Integer ALU reg-reg operation with extract
6204 // As for _vshift above, but result generated in EX2
6205 // Eg. EXTR x0, x1, x2, #N
6206 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6207 %{
6208 single_instruction;
6209 dst : EX2(write);
6210 src1 : ISS(read);
6211 src2 : ISS(read);
6212 INS1 : ISS; // Can only dual issue as Instruction 1
6213 ALU : EX1;
6214 %}
6215
6216 // Integer ALU reg operation
6217 // Eg. NEG x0, x1
6218 pipe_class ialu_reg(iRegI dst, iRegI src)
6219 %{
6220 single_instruction;
6221 dst : EX2(write);
6222 src : EX1(read);
6223 INS01 : ISS;
6224 ALU : EX2;
6225 %}
6226
6227 // Integer ALU reg mmediate operation
6228 // Eg. ADD x0, x1, #N
6229 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6230 %{
6231 single_instruction;
6232 dst : EX2(write);
6233 src1 : EX1(read);
6234 INS01 : ISS;
6235 ALU : EX2;
6236 %}
6237
6238 // Integer ALU immediate operation (no source operands)
6239 // Eg. MOV x0, #N
6240 pipe_class ialu_imm(iRegI dst)
6241 %{
6242 single_instruction;
6243 dst : EX1(write);
6244 INS01 : ISS;
6245 ALU : EX1;
6246 %}
6247
6248 //------- Compare operation -------------------------------
6249
6250 // Compare reg-reg
6251 // Eg. CMP x0, x1
6252 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6253 %{
6254 single_instruction;
6255 // fixed_latency(16);
6256 cr : EX2(write);
6257 op1 : EX1(read);
6258 op2 : EX1(read);
6259 INS01 : ISS;
6260 ALU : EX2;
6261 %}
6262
6263 // Compare reg-reg
6264 // Eg. CMP x0, #N
6265 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6266 %{
6267 single_instruction;
6268 // fixed_latency(16);
6269 cr : EX2(write);
6270 op1 : EX1(read);
6271 INS01 : ISS;
6272 ALU : EX2;
6273 %}
6274
6275 //------- Conditional instructions ------------------------
6276
6277 // Conditional no operands
6278 // Eg. CSINC x0, zr, zr, <cond>
6279 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6280 %{
6281 single_instruction;
6282 cr : EX1(read);
6283 dst : EX2(write);
6284 INS01 : ISS;
6285 ALU : EX2;
6286 %}
6287
6288 // Conditional 2 operand
6289 // EG. CSEL X0, X1, X2, <cond>
6290 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6291 %{
6292 single_instruction;
6293 cr : EX1(read);
6294 src1 : EX1(read);
6295 src2 : EX1(read);
6296 dst : EX2(write);
6297 INS01 : ISS;
6298 ALU : EX2;
6299 %}
6300
6301 // Conditional 2 operand
6302 // EG. CSEL X0, X1, X2, <cond>
6303 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6304 %{
6305 single_instruction;
6306 cr : EX1(read);
6307 src : EX1(read);
6308 dst : EX2(write);
6309 INS01 : ISS;
6310 ALU : EX2;
6311 %}
6312
6313 //------- Multiply pipeline operations --------------------
6314
6315 // Multiply reg-reg
6316 // Eg. MUL w0, w1, w2
6317 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6318 %{
6319 single_instruction;
6320 dst : WR(write);
6321 src1 : ISS(read);
6322 src2 : ISS(read);
6323 INS01 : ISS;
6324 MAC : WR;
6325 %}
6326
6327 // Multiply accumulate
6328 // Eg. MADD w0, w1, w2, w3
6329 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6330 %{
6331 single_instruction;
6332 dst : WR(write);
6333 src1 : ISS(read);
6334 src2 : ISS(read);
6335 src3 : ISS(read);
6336 INS01 : ISS;
6337 MAC : WR;
6338 %}
6339
6340 // Eg. MUL w0, w1, w2
6341 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6342 %{
6343 single_instruction;
6344 fixed_latency(3); // Maximum latency for 64 bit mul
6345 dst : WR(write);
6346 src1 : ISS(read);
6347 src2 : ISS(read);
6348 INS01 : ISS;
6349 MAC : WR;
6350 %}
6351
6352 // Multiply accumulate
6353 // Eg. MADD w0, w1, w2, w3
6354 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6355 %{
6356 single_instruction;
6357 fixed_latency(3); // Maximum latency for 64 bit mul
6358 dst : WR(write);
6359 src1 : ISS(read);
6360 src2 : ISS(read);
6361 src3 : ISS(read);
6362 INS01 : ISS;
6363 MAC : WR;
6364 %}
6365
6366 //------- Divide pipeline operations --------------------
6367
6368 // Eg. SDIV w0, w1, w2
6369 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6370 %{
6371 single_instruction;
6372 fixed_latency(8); // Maximum latency for 32 bit divide
6373 dst : WR(write);
6374 src1 : ISS(read);
6375 src2 : ISS(read);
6376 INS0 : ISS; // Can only dual issue as instruction 0
6377 DIV : WR;
6378 %}
6379
6380 // Eg. SDIV x0, x1, x2
6381 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6382 %{
6383 single_instruction;
6384 fixed_latency(16); // Maximum latency for 64 bit divide
6385 dst : WR(write);
6386 src1 : ISS(read);
6387 src2 : ISS(read);
6388 INS0 : ISS; // Can only dual issue as instruction 0
6389 DIV : WR;
6390 %}
6391
6392 //------- Load pipeline operations ------------------------
6393
6394 // Load - prefetch
6395 // Eg. PFRM <mem>
6396 pipe_class iload_prefetch(memory mem)
6397 %{
6398 single_instruction;
6399 mem : ISS(read);
6400 INS01 : ISS;
6401 LDST : WR;
6402 %}
6403
6404 // Load - reg, mem
6405 // Eg. LDR x0, <mem>
6406 pipe_class iload_reg_mem(iRegI dst, memory mem)
6407 %{
6408 single_instruction;
6409 dst : WR(write);
6410 mem : ISS(read);
6411 INS01 : ISS;
6412 LDST : WR;
6413 %}
6414
6415 // Load - reg, reg
6416 // Eg. LDR x0, [sp, x1]
6417 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6418 %{
6419 single_instruction;
6420 dst : WR(write);
6421 src : ISS(read);
6422 INS01 : ISS;
6423 LDST : WR;
6424 %}
6425
6426 //------- Store pipeline operations -----------------------
6427
6428 // Store - zr, mem
6429 // Eg. STR zr, <mem>
6430 pipe_class istore_mem(memory mem)
6431 %{
6432 single_instruction;
6433 mem : ISS(read);
6434 INS01 : ISS;
6435 LDST : WR;
6436 %}
6437
6438 // Store - reg, mem
6439 // Eg. STR x0, <mem>
6440 pipe_class istore_reg_mem(iRegI src, memory mem)
6441 %{
6442 single_instruction;
6443 mem : ISS(read);
6444 src : EX2(read);
6445 INS01 : ISS;
6446 LDST : WR;
6447 %}
6448
6449 // Store - reg, reg
6450 // Eg. STR x0, [sp, x1]
6451 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6452 %{
6453 single_instruction;
6454 dst : ISS(read);
6455 src : EX2(read);
6456 INS01 : ISS;
6457 LDST : WR;
6458 %}
6459
6460 //------- Store pipeline operations -----------------------
6461
6462 // Branch
6463 pipe_class pipe_branch()
6464 %{
6465 single_instruction;
6466 INS01 : ISS;
6467 BRANCH : EX1;
6468 %}
6469
6470 // Conditional branch
6471 pipe_class pipe_branch_cond(rFlagsReg cr)
6472 %{
6473 single_instruction;
6474 cr : EX1(read);
6475 INS01 : ISS;
6476 BRANCH : EX1;
6477 %}
6478
6479 // Compare & Branch
6480 // EG. CBZ/CBNZ
6481 pipe_class pipe_cmp_branch(iRegI op1)
6482 %{
6483 single_instruction;
6484 op1 : EX1(read);
6485 INS01 : ISS;
6486 BRANCH : EX1;
6487 %}
6488
6489 //------- Synchronisation operations ----------------------
6490
6491 // Any operation requiring serialization.
6492 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6493 pipe_class pipe_serial()
6494 %{
6495 single_instruction;
6496 force_serialization;
6497 fixed_latency(16);
6498 INS01 : ISS(2); // Cannot dual issue with any other instruction
6499 LDST : WR;
6500 %}
6501
6502 // Generic big/slow expanded idiom - also serialized
6503 pipe_class pipe_slow()
6504 %{
6505 instruction_count(10);
6506 multiple_bundles;
6507 force_serialization;
6508 fixed_latency(16);
6509 INS01 : ISS(2); // Cannot dual issue with any other instruction
6510 LDST : WR;
6511 %}
6512
6513 // Empty pipeline class
6514 pipe_class pipe_class_empty()
6515 %{
6516 single_instruction;
6517 fixed_latency(0);
6518 %}
6519
6520 // Default pipeline class.
6521 pipe_class pipe_class_default()
6522 %{
6523 single_instruction;
6524 fixed_latency(2);
6525 %}
6526
6527 // Pipeline class for compares.
6528 pipe_class pipe_class_compare()
6529 %{
6530 single_instruction;
6531 fixed_latency(16);
6532 %}
6533
6534 // Pipeline class for memory operations.
6535 pipe_class pipe_class_memory()
6536 %{
6537 single_instruction;
6538 fixed_latency(16);
6539 %}
6540
6541 // Pipeline class for call.
6542 pipe_class pipe_class_call()
6543 %{
6544 single_instruction;
6545 fixed_latency(100);
6546 %}
6547
6548 // Define the class for the Nop node.
6549 define %{
6550 MachNop = pipe_class_empty;
6551 %}
6552
6553 %}
6554 //----------INSTRUCTIONS-------------------------------------------------------
6555 //
6556 // match -- States which machine-independent subtree may be replaced
6557 // by this instruction.
6558 // ins_cost -- The estimated cost of this instruction is used by instruction
6559 // selection to identify a minimum cost tree of machine
6560 // instructions that matches a tree of machine-independent
6561 // instructions.
6562 // format -- A string providing the disassembly for this instruction.
6563 // The value of an instruction's operand may be inserted
6564 // by referring to it with a '$' prefix.
6565 // opcode -- Three instruction opcodes may be provided. These are referred
6566 // to within an encode class as $primary, $secondary, and $tertiary
6567 // rrspectively. The primary opcode is commonly used to
6568 // indicate the type of machine instruction, while secondary
6569 // and tertiary are often used for prefix options or addressing
6570 // modes.
6571 // ins_encode -- A list of encode classes with parameters. The encode class
6572 // name must have been defined in an 'enc_class' specification
6573 // in the encode section of the architecture description.
6574
6575 // ============================================================================
6576 // Memory (Load/Store) Instructions
6577
6578 // Load Instructions
6579
6580 // Load Byte (8 bit signed)
6581 instruct loadB(iRegINoSp dst, memory1 mem)
6582 %{
6583 match(Set dst (LoadB mem));
6584 predicate(!needs_acquiring_load(n));
6585
6586 ins_cost(4 * INSN_COST);
6587 format %{ "ldrsbw $dst, $mem\t# byte" %}
6588
6589 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6590
6591 ins_pipe(iload_reg_mem);
6592 %}
6593
6594 // Load Byte (8 bit signed) into long
6595 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6596 %{
6597 match(Set dst (ConvI2L (LoadB mem)));
6598 predicate(!needs_acquiring_load(n->in(1)));
6599
6600 ins_cost(4 * INSN_COST);
6601 format %{ "ldrsb $dst, $mem\t# byte" %}
6602
6603 ins_encode(aarch64_enc_ldrsb(dst, mem));
6604
6605 ins_pipe(iload_reg_mem);
6606 %}
6607
6608 // Load Byte (8 bit unsigned)
6609 instruct loadUB(iRegINoSp dst, memory1 mem)
6610 %{
6611 match(Set dst (LoadUB mem));
6612 predicate(!needs_acquiring_load(n));
6613
6614 ins_cost(4 * INSN_COST);
6615 format %{ "ldrbw $dst, $mem\t# byte" %}
6616
6617 ins_encode(aarch64_enc_ldrb(dst, mem));
6618
6619 ins_pipe(iload_reg_mem);
6620 %}
6621
6622 // Load Byte (8 bit unsigned) into long
6623 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6624 %{
6625 match(Set dst (ConvI2L (LoadUB mem)));
6626 predicate(!needs_acquiring_load(n->in(1)));
6627
6628 ins_cost(4 * INSN_COST);
6629 format %{ "ldrb $dst, $mem\t# byte" %}
6630
6631 ins_encode(aarch64_enc_ldrb(dst, mem));
6632
6633 ins_pipe(iload_reg_mem);
6634 %}
6635
6636 // Load Short (16 bit signed)
6637 instruct loadS(iRegINoSp dst, memory2 mem)
6638 %{
6639 match(Set dst (LoadS mem));
6640 predicate(!needs_acquiring_load(n));
6641
6642 ins_cost(4 * INSN_COST);
6643 format %{ "ldrshw $dst, $mem\t# short" %}
6644
6645 ins_encode(aarch64_enc_ldrshw(dst, mem));
6646
6647 ins_pipe(iload_reg_mem);
6648 %}
6649
6650 // Load Short (16 bit signed) into long
6651 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6652 %{
6653 match(Set dst (ConvI2L (LoadS mem)));
6654 predicate(!needs_acquiring_load(n->in(1)));
6655
6656 ins_cost(4 * INSN_COST);
6657 format %{ "ldrsh $dst, $mem\t# short" %}
6658
6659 ins_encode(aarch64_enc_ldrsh(dst, mem));
6660
6661 ins_pipe(iload_reg_mem);
6662 %}
6663
6664 // Load Char (16 bit unsigned)
6665 instruct loadUS(iRegINoSp dst, memory2 mem)
6666 %{
6667 match(Set dst (LoadUS mem));
6668 predicate(!needs_acquiring_load(n));
6669
6670 ins_cost(4 * INSN_COST);
6671 format %{ "ldrh $dst, $mem\t# short" %}
6672
6673 ins_encode(aarch64_enc_ldrh(dst, mem));
6674
6675 ins_pipe(iload_reg_mem);
6676 %}
6677
6678 // Load Short/Char (16 bit unsigned) into long
6679 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6680 %{
6681 match(Set dst (ConvI2L (LoadUS mem)));
6682 predicate(!needs_acquiring_load(n->in(1)));
6683
6684 ins_cost(4 * INSN_COST);
6685 format %{ "ldrh $dst, $mem\t# short" %}
6686
6687 ins_encode(aarch64_enc_ldrh(dst, mem));
6688
6689 ins_pipe(iload_reg_mem);
6690 %}
6691
6692 // Load Integer (32 bit signed)
6693 instruct loadI(iRegINoSp dst, memory4 mem)
6694 %{
6695 match(Set dst (LoadI mem));
6696 predicate(!needs_acquiring_load(n));
6697
6698 ins_cost(4 * INSN_COST);
6699 format %{ "ldrw $dst, $mem\t# int" %}
6700
6701 ins_encode(aarch64_enc_ldrw(dst, mem));
6702
6703 ins_pipe(iload_reg_mem);
6704 %}
6705
6706 // Load Integer (32 bit signed) into long
6707 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6708 %{
6709 match(Set dst (ConvI2L (LoadI mem)));
6710 predicate(!needs_acquiring_load(n->in(1)));
6711
6712 ins_cost(4 * INSN_COST);
6713 format %{ "ldrsw $dst, $mem\t# int" %}
6714
6715 ins_encode(aarch64_enc_ldrsw(dst, mem));
6716
6717 ins_pipe(iload_reg_mem);
6718 %}
6719
6720 // Load Integer (32 bit unsigned) into long
6721 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6722 %{
6723 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6724 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6725
6726 ins_cost(4 * INSN_COST);
6727 format %{ "ldrw $dst, $mem\t# int" %}
6728
6729 ins_encode(aarch64_enc_ldrw(dst, mem));
6730
6731 ins_pipe(iload_reg_mem);
6732 %}
6733
6734 // Load Long (64 bit signed)
6735 instruct loadL(iRegLNoSp dst, memory8 mem)
6736 %{
6737 match(Set dst (LoadL mem));
6738 predicate(!needs_acquiring_load(n));
6739
6740 ins_cost(4 * INSN_COST);
6741 format %{ "ldr $dst, $mem\t# int" %}
6742
6743 ins_encode(aarch64_enc_ldr(dst, mem));
6744
6745 ins_pipe(iload_reg_mem);
6746 %}
6747
6748 // Load Range
6749 instruct loadRange(iRegINoSp dst, memory4 mem)
6750 %{
6751 match(Set dst (LoadRange mem));
6752
6753 ins_cost(4 * INSN_COST);
6754 format %{ "ldrw $dst, $mem\t# range" %}
6755
6756 ins_encode(aarch64_enc_ldrw(dst, mem));
6757
6758 ins_pipe(iload_reg_mem);
6759 %}
6760
6761 // Load Pointer
6762 instruct loadP(iRegPNoSp dst, memory8 mem)
6763 %{
6764 match(Set dst (LoadP mem));
6765 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6766
6767 ins_cost(4 * INSN_COST);
6768 format %{ "ldr $dst, $mem\t# ptr" %}
6769
6770 ins_encode(aarch64_enc_ldr(dst, mem));
6771
6772 ins_pipe(iload_reg_mem);
6773 %}
6774
6775 // Load Compressed Pointer
6776 instruct loadN(iRegNNoSp dst, memory4 mem)
6777 %{
6778 match(Set dst (LoadN mem));
6779 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6780
6781 ins_cost(4 * INSN_COST);
6782 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6783
6784 ins_encode(aarch64_enc_ldrw(dst, mem));
6785
6786 ins_pipe(iload_reg_mem);
6787 %}
6788
6789 // Load Klass Pointer
6790 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6791 %{
6792 match(Set dst (LoadKlass mem));
6793 predicate(!needs_acquiring_load(n));
6794
6795 ins_cost(4 * INSN_COST);
6796 format %{ "ldr $dst, $mem\t# class" %}
6797
6798 ins_encode(aarch64_enc_ldr(dst, mem));
6799
6800 ins_pipe(iload_reg_mem);
6801 %}
6802
6803 // Load Narrow Klass Pointer
6804 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6805 %{
6806 match(Set dst (LoadNKlass mem));
6807 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6808
6809 ins_cost(4 * INSN_COST);
6810 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6811
6812 ins_encode(aarch64_enc_ldrw(dst, mem));
6813
6814 ins_pipe(iload_reg_mem);
6815 %}
6816
6817 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6818 %{
6819 match(Set dst (LoadNKlass mem));
6820 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6821
6822 ins_cost(4 * INSN_COST);
6823 format %{
6824 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6825 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6826 %}
6827 ins_encode %{
6828 // inlined aarch64_enc_ldrw
6829 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6830 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6831 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6832 %}
6833 ins_pipe(iload_reg_mem);
6834 %}
6835
6836 // Load Float
6837 instruct loadF(vRegF dst, memory4 mem)
6838 %{
6839 match(Set dst (LoadF mem));
6840 predicate(!needs_acquiring_load(n));
6841
6842 ins_cost(4 * INSN_COST);
6843 format %{ "ldrs $dst, $mem\t# float" %}
6844
6845 ins_encode( aarch64_enc_ldrs(dst, mem) );
6846
6847 ins_pipe(pipe_class_memory);
6848 %}
6849
6850 // Load Double
6851 instruct loadD(vRegD dst, memory8 mem)
6852 %{
6853 match(Set dst (LoadD mem));
6854 predicate(!needs_acquiring_load(n));
6855
6856 ins_cost(4 * INSN_COST);
6857 format %{ "ldrd $dst, $mem\t# double" %}
6858
6859 ins_encode( aarch64_enc_ldrd(dst, mem) );
6860
6861 ins_pipe(pipe_class_memory);
6862 %}
6863
6864
6865 // Load Int Constant
6866 instruct loadConI(iRegINoSp dst, immI src)
6867 %{
6868 match(Set dst src);
6869
6870 ins_cost(INSN_COST);
6871 format %{ "mov $dst, $src\t# int" %}
6872
6873 ins_encode( aarch64_enc_movw_imm(dst, src) );
6874
6875 ins_pipe(ialu_imm);
6876 %}
6877
6878 // Load Long Constant
6879 instruct loadConL(iRegLNoSp dst, immL src)
6880 %{
6881 match(Set dst src);
6882
6883 ins_cost(INSN_COST);
6884 format %{ "mov $dst, $src\t# long" %}
6885
6886 ins_encode( aarch64_enc_mov_imm(dst, src) );
6887
6888 ins_pipe(ialu_imm);
6889 %}
6890
6891 // Load Pointer Constant
6892
6893 instruct loadConP(iRegPNoSp dst, immP con)
6894 %{
6895 match(Set dst con);
6896
6897 ins_cost(INSN_COST * 4);
6898 format %{
6899 "mov $dst, $con\t# ptr"
6900 %}
6901
6902 ins_encode(aarch64_enc_mov_p(dst, con));
6903
6904 ins_pipe(ialu_imm);
6905 %}
6906
6907 // Load Null Pointer Constant
6908
6909 instruct loadConP0(iRegPNoSp dst, immP0 con)
6910 %{
6911 match(Set dst con);
6912
6913 ins_cost(INSN_COST);
6914 format %{ "mov $dst, $con\t# nullptr ptr" %}
6915
6916 ins_encode(aarch64_enc_mov_p0(dst, con));
6917
6918 ins_pipe(ialu_imm);
6919 %}
6920
6921 // Load Pointer Constant One
6922
6923 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6924 %{
6925 match(Set dst con);
6926
6927 ins_cost(INSN_COST);
6928 format %{ "mov $dst, $con\t# nullptr ptr" %}
6929
6930 ins_encode(aarch64_enc_mov_p1(dst, con));
6931
6932 ins_pipe(ialu_imm);
6933 %}
6934
6935 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6936 %{
6937 match(Set dst con);
6938
6939 ins_cost(INSN_COST);
6940 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6941
6942 ins_encode %{
6943 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6944 %}
6945
6946 ins_pipe(ialu_imm);
6947 %}
6948
6949 // Load Narrow Pointer Constant
6950
6951 instruct loadConN(iRegNNoSp dst, immN con)
6952 %{
6953 match(Set dst con);
6954
6955 ins_cost(INSN_COST * 4);
6956 format %{ "mov $dst, $con\t# compressed ptr" %}
6957
6958 ins_encode(aarch64_enc_mov_n(dst, con));
6959
6960 ins_pipe(ialu_imm);
6961 %}
6962
6963 // Load Narrow Null Pointer Constant
6964
6965 instruct loadConN0(iRegNNoSp dst, immN0 con)
6966 %{
6967 match(Set dst con);
6968
6969 ins_cost(INSN_COST);
6970 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6971
6972 ins_encode(aarch64_enc_mov_n0(dst, con));
6973
6974 ins_pipe(ialu_imm);
6975 %}
6976
6977 // Load Narrow Klass Constant
6978
6979 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6980 %{
6981 match(Set dst con);
6982
6983 ins_cost(INSN_COST);
6984 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6985
6986 ins_encode(aarch64_enc_mov_nk(dst, con));
6987
6988 ins_pipe(ialu_imm);
6989 %}
6990
6991 // Load Packed Float Constant
6992
6993 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6994 match(Set dst con);
6995 ins_cost(INSN_COST * 4);
6996 format %{ "fmovs $dst, $con"%}
6997 ins_encode %{
6998 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6999 %}
7000
7001 ins_pipe(fp_imm_s);
7002 %}
7003
7004 // Load Float Constant
7005
7006 instruct loadConF(vRegF dst, immF con) %{
7007 match(Set dst con);
7008
7009 ins_cost(INSN_COST * 4);
7010
7011 format %{
7012 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7013 %}
7014
7015 ins_encode %{
7016 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7017 %}
7018
7019 ins_pipe(fp_load_constant_s);
7020 %}
7021
7022 // Load Packed Double Constant
7023
7024 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7025 match(Set dst con);
7026 ins_cost(INSN_COST);
7027 format %{ "fmovd $dst, $con"%}
7028 ins_encode %{
7029 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7030 %}
7031
7032 ins_pipe(fp_imm_d);
7033 %}
7034
7035 // Load Double Constant
7036
7037 instruct loadConD(vRegD dst, immD con) %{
7038 match(Set dst con);
7039
7040 ins_cost(INSN_COST * 5);
7041 format %{
7042 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7043 %}
7044
7045 ins_encode %{
7046 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7047 %}
7048
7049 ins_pipe(fp_load_constant_d);
7050 %}
7051
7052 // Load Half Float Constant
7053 instruct loadConH(vRegF dst, immH con) %{
7054 match(Set dst con);
7055 format %{ "mov rscratch1, $con\n\t"
7056 "fmov $dst, rscratch1"
7057 %}
7058 ins_encode %{
7059 __ movw(rscratch1, (uint32_t)$con$$constant);
7060 __ fmovs($dst$$FloatRegister, rscratch1);
7061 %}
7062 ins_pipe(pipe_class_default);
7063 %}
7064
7065 // Store Instructions
7066
7067 // Store Byte
7068 instruct storeB(iRegIorL2I src, memory1 mem)
7069 %{
7070 match(Set mem (StoreB mem src));
7071 predicate(!needs_releasing_store(n));
7072
7073 ins_cost(INSN_COST);
7074 format %{ "strb $src, $mem\t# byte" %}
7075
7076 ins_encode(aarch64_enc_strb(src, mem));
7077
7078 ins_pipe(istore_reg_mem);
7079 %}
7080
7081
7082 instruct storeimmB0(immI0 zero, memory1 mem)
7083 %{
7084 match(Set mem (StoreB mem zero));
7085 predicate(!needs_releasing_store(n));
7086
7087 ins_cost(INSN_COST);
7088 format %{ "strb rscractch2, $mem\t# byte" %}
7089
7090 ins_encode(aarch64_enc_strb0(mem));
7091
7092 ins_pipe(istore_mem);
7093 %}
7094
7095 // Store Char/Short
7096 instruct storeC(iRegIorL2I src, memory2 mem)
7097 %{
7098 match(Set mem (StoreC mem src));
7099 predicate(!needs_releasing_store(n));
7100
7101 ins_cost(INSN_COST);
7102 format %{ "strh $src, $mem\t# short" %}
7103
7104 ins_encode(aarch64_enc_strh(src, mem));
7105
7106 ins_pipe(istore_reg_mem);
7107 %}
7108
7109 instruct storeimmC0(immI0 zero, memory2 mem)
7110 %{
7111 match(Set mem (StoreC mem zero));
7112 predicate(!needs_releasing_store(n));
7113
7114 ins_cost(INSN_COST);
7115 format %{ "strh zr, $mem\t# short" %}
7116
7117 ins_encode(aarch64_enc_strh0(mem));
7118
7119 ins_pipe(istore_mem);
7120 %}
7121
7122 // Store Integer
7123
7124 instruct storeI(iRegIorL2I src, memory4 mem)
7125 %{
7126 match(Set mem(StoreI mem src));
7127 predicate(!needs_releasing_store(n));
7128
7129 ins_cost(INSN_COST);
7130 format %{ "strw $src, $mem\t# int" %}
7131
7132 ins_encode(aarch64_enc_strw(src, mem));
7133
7134 ins_pipe(istore_reg_mem);
7135 %}
7136
7137 instruct storeimmI0(immI0 zero, memory4 mem)
7138 %{
7139 match(Set mem(StoreI mem zero));
7140 predicate(!needs_releasing_store(n));
7141
7142 ins_cost(INSN_COST);
7143 format %{ "strw zr, $mem\t# int" %}
7144
7145 ins_encode(aarch64_enc_strw0(mem));
7146
7147 ins_pipe(istore_mem);
7148 %}
7149
7150 // Store Long (64 bit signed)
7151 instruct storeL(iRegL src, memory8 mem)
7152 %{
7153 match(Set mem (StoreL mem src));
7154 predicate(!needs_releasing_store(n));
7155
7156 ins_cost(INSN_COST);
7157 format %{ "str $src, $mem\t# int" %}
7158
7159 ins_encode(aarch64_enc_str(src, mem));
7160
7161 ins_pipe(istore_reg_mem);
7162 %}
7163
7164 // Store Long (64 bit signed)
7165 instruct storeimmL0(immL0 zero, memory8 mem)
7166 %{
7167 match(Set mem (StoreL mem zero));
7168 predicate(!needs_releasing_store(n));
7169
7170 ins_cost(INSN_COST);
7171 format %{ "str zr, $mem\t# int" %}
7172
7173 ins_encode(aarch64_enc_str0(mem));
7174
7175 ins_pipe(istore_mem);
7176 %}
7177
7178 // Store Pointer
7179 instruct storeP(iRegP src, memory8 mem)
7180 %{
7181 match(Set mem (StoreP mem src));
7182 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7183
7184 ins_cost(INSN_COST);
7185 format %{ "str $src, $mem\t# ptr" %}
7186
7187 ins_encode(aarch64_enc_str(src, mem));
7188
7189 ins_pipe(istore_reg_mem);
7190 %}
7191
7192 // Store Pointer
7193 instruct storeimmP0(immP0 zero, memory8 mem)
7194 %{
7195 match(Set mem (StoreP mem zero));
7196 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7197
7198 ins_cost(INSN_COST);
7199 format %{ "str zr, $mem\t# ptr" %}
7200
7201 ins_encode(aarch64_enc_str0(mem));
7202
7203 ins_pipe(istore_mem);
7204 %}
7205
7206 // Store Compressed Pointer
7207 instruct storeN(iRegN src, memory4 mem)
7208 %{
7209 match(Set mem (StoreN mem src));
7210 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7211
7212 ins_cost(INSN_COST);
7213 format %{ "strw $src, $mem\t# compressed ptr" %}
7214
7215 ins_encode(aarch64_enc_strw(src, mem));
7216
7217 ins_pipe(istore_reg_mem);
7218 %}
7219
7220 instruct storeImmN0(immN0 zero, memory4 mem)
7221 %{
7222 match(Set mem (StoreN mem zero));
7223 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7224
7225 ins_cost(INSN_COST);
7226 format %{ "strw zr, $mem\t# compressed ptr" %}
7227
7228 ins_encode(aarch64_enc_strw0(mem));
7229
7230 ins_pipe(istore_mem);
7231 %}
7232
7233 // Store Float
7234 instruct storeF(vRegF src, memory4 mem)
7235 %{
7236 match(Set mem (StoreF mem src));
7237 predicate(!needs_releasing_store(n));
7238
7239 ins_cost(INSN_COST);
7240 format %{ "strs $src, $mem\t# float" %}
7241
7242 ins_encode( aarch64_enc_strs(src, mem) );
7243
7244 ins_pipe(pipe_class_memory);
7245 %}
7246
7247 // TODO
7248 // implement storeImmF0 and storeFImmPacked
7249
7250 // Store Double
7251 instruct storeD(vRegD src, memory8 mem)
7252 %{
7253 match(Set mem (StoreD mem src));
7254 predicate(!needs_releasing_store(n));
7255
7256 ins_cost(INSN_COST);
7257 format %{ "strd $src, $mem\t# double" %}
7258
7259 ins_encode( aarch64_enc_strd(src, mem) );
7260
7261 ins_pipe(pipe_class_memory);
7262 %}
7263
7264 // Store Compressed Klass Pointer
7265 instruct storeNKlass(iRegN src, memory4 mem)
7266 %{
7267 predicate(!needs_releasing_store(n));
7268 match(Set mem (StoreNKlass mem src));
7269
7270 ins_cost(INSN_COST);
7271 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7272
7273 ins_encode(aarch64_enc_strw(src, mem));
7274
7275 ins_pipe(istore_reg_mem);
7276 %}
7277
7278 // TODO
7279 // implement storeImmD0 and storeDImmPacked
7280
7281 // prefetch instructions
7282 // Must be safe to execute with invalid address (cannot fault).
7283
7284 instruct prefetchalloc( memory8 mem ) %{
7285 match(PrefetchAllocation mem);
7286
7287 ins_cost(INSN_COST);
7288 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7289
7290 ins_encode( aarch64_enc_prefetchw(mem) );
7291
7292 ins_pipe(iload_prefetch);
7293 %}
7294
7295 // ---------------- volatile loads and stores ----------------
7296
7297 // Load Byte (8 bit signed)
7298 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7299 %{
7300 match(Set dst (LoadB mem));
7301
7302 ins_cost(VOLATILE_REF_COST);
7303 format %{ "ldarsb $dst, $mem\t# byte" %}
7304
7305 ins_encode(aarch64_enc_ldarsb(dst, mem));
7306
7307 ins_pipe(pipe_serial);
7308 %}
7309
7310 // Load Byte (8 bit signed) into long
7311 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7312 %{
7313 match(Set dst (ConvI2L (LoadB mem)));
7314
7315 ins_cost(VOLATILE_REF_COST);
7316 format %{ "ldarsb $dst, $mem\t# byte" %}
7317
7318 ins_encode(aarch64_enc_ldarsb(dst, mem));
7319
7320 ins_pipe(pipe_serial);
7321 %}
7322
7323 // Load Byte (8 bit unsigned)
7324 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7325 %{
7326 match(Set dst (LoadUB mem));
7327
7328 ins_cost(VOLATILE_REF_COST);
7329 format %{ "ldarb $dst, $mem\t# byte" %}
7330
7331 ins_encode(aarch64_enc_ldarb(dst, mem));
7332
7333 ins_pipe(pipe_serial);
7334 %}
7335
7336 // Load Byte (8 bit unsigned) into long
7337 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7338 %{
7339 match(Set dst (ConvI2L (LoadUB mem)));
7340
7341 ins_cost(VOLATILE_REF_COST);
7342 format %{ "ldarb $dst, $mem\t# byte" %}
7343
7344 ins_encode(aarch64_enc_ldarb(dst, mem));
7345
7346 ins_pipe(pipe_serial);
7347 %}
7348
7349 // Load Short (16 bit signed)
7350 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7351 %{
7352 match(Set dst (LoadS mem));
7353
7354 ins_cost(VOLATILE_REF_COST);
7355 format %{ "ldarshw $dst, $mem\t# short" %}
7356
7357 ins_encode(aarch64_enc_ldarshw(dst, mem));
7358
7359 ins_pipe(pipe_serial);
7360 %}
7361
7362 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7363 %{
7364 match(Set dst (LoadUS mem));
7365
7366 ins_cost(VOLATILE_REF_COST);
7367 format %{ "ldarhw $dst, $mem\t# short" %}
7368
7369 ins_encode(aarch64_enc_ldarhw(dst, mem));
7370
7371 ins_pipe(pipe_serial);
7372 %}
7373
7374 // Load Short/Char (16 bit unsigned) into long
7375 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7376 %{
7377 match(Set dst (ConvI2L (LoadUS mem)));
7378
7379 ins_cost(VOLATILE_REF_COST);
7380 format %{ "ldarh $dst, $mem\t# short" %}
7381
7382 ins_encode(aarch64_enc_ldarh(dst, mem));
7383
7384 ins_pipe(pipe_serial);
7385 %}
7386
7387 // Load Short/Char (16 bit signed) into long
7388 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7389 %{
7390 match(Set dst (ConvI2L (LoadS mem)));
7391
7392 ins_cost(VOLATILE_REF_COST);
7393 format %{ "ldarh $dst, $mem\t# short" %}
7394
7395 ins_encode(aarch64_enc_ldarsh(dst, mem));
7396
7397 ins_pipe(pipe_serial);
7398 %}
7399
7400 // Load Integer (32 bit signed)
7401 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7402 %{
7403 match(Set dst (LoadI mem));
7404
7405 ins_cost(VOLATILE_REF_COST);
7406 format %{ "ldarw $dst, $mem\t# int" %}
7407
7408 ins_encode(aarch64_enc_ldarw(dst, mem));
7409
7410 ins_pipe(pipe_serial);
7411 %}
7412
7413 // Load Integer (32 bit unsigned) into long
7414 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7415 %{
7416 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7417
7418 ins_cost(VOLATILE_REF_COST);
7419 format %{ "ldarw $dst, $mem\t# int" %}
7420
7421 ins_encode(aarch64_enc_ldarw(dst, mem));
7422
7423 ins_pipe(pipe_serial);
7424 %}
7425
7426 // Load Long (64 bit signed)
7427 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7428 %{
7429 match(Set dst (LoadL mem));
7430
7431 ins_cost(VOLATILE_REF_COST);
7432 format %{ "ldar $dst, $mem\t# int" %}
7433
7434 ins_encode(aarch64_enc_ldar(dst, mem));
7435
7436 ins_pipe(pipe_serial);
7437 %}
7438
7439 // Load Pointer
7440 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7441 %{
7442 match(Set dst (LoadP mem));
7443 predicate(n->as_Load()->barrier_data() == 0);
7444
7445 ins_cost(VOLATILE_REF_COST);
7446 format %{ "ldar $dst, $mem\t# ptr" %}
7447
7448 ins_encode(aarch64_enc_ldar(dst, mem));
7449
7450 ins_pipe(pipe_serial);
7451 %}
7452
7453 // Load Compressed Pointer
7454 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7455 %{
7456 match(Set dst (LoadN mem));
7457 predicate(n->as_Load()->barrier_data() == 0);
7458
7459 ins_cost(VOLATILE_REF_COST);
7460 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7461
7462 ins_encode(aarch64_enc_ldarw(dst, mem));
7463
7464 ins_pipe(pipe_serial);
7465 %}
7466
7467 // Load Float
7468 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7469 %{
7470 match(Set dst (LoadF mem));
7471
7472 ins_cost(VOLATILE_REF_COST);
7473 format %{ "ldars $dst, $mem\t# float" %}
7474
7475 ins_encode( aarch64_enc_fldars(dst, mem) );
7476
7477 ins_pipe(pipe_serial);
7478 %}
7479
7480 // Load Double
7481 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7482 %{
7483 match(Set dst (LoadD mem));
7484
7485 ins_cost(VOLATILE_REF_COST);
7486 format %{ "ldard $dst, $mem\t# double" %}
7487
7488 ins_encode( aarch64_enc_fldard(dst, mem) );
7489
7490 ins_pipe(pipe_serial);
7491 %}
7492
7493 // Store Byte
7494 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7495 %{
7496 match(Set mem (StoreB mem src));
7497
7498 ins_cost(VOLATILE_REF_COST);
7499 format %{ "stlrb $src, $mem\t# byte" %}
7500
7501 ins_encode(aarch64_enc_stlrb(src, mem));
7502
7503 ins_pipe(pipe_class_memory);
7504 %}
7505
7506 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7507 %{
7508 match(Set mem (StoreB mem zero));
7509
7510 ins_cost(VOLATILE_REF_COST);
7511 format %{ "stlrb zr, $mem\t# byte" %}
7512
7513 ins_encode(aarch64_enc_stlrb0(mem));
7514
7515 ins_pipe(pipe_class_memory);
7516 %}
7517
7518 // Store Char/Short
7519 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7520 %{
7521 match(Set mem (StoreC mem src));
7522
7523 ins_cost(VOLATILE_REF_COST);
7524 format %{ "stlrh $src, $mem\t# short" %}
7525
7526 ins_encode(aarch64_enc_stlrh(src, mem));
7527
7528 ins_pipe(pipe_class_memory);
7529 %}
7530
7531 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7532 %{
7533 match(Set mem (StoreC mem zero));
7534
7535 ins_cost(VOLATILE_REF_COST);
7536 format %{ "stlrh zr, $mem\t# short" %}
7537
7538 ins_encode(aarch64_enc_stlrh0(mem));
7539
7540 ins_pipe(pipe_class_memory);
7541 %}
7542
7543 // Store Integer
7544
7545 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7546 %{
7547 match(Set mem(StoreI mem src));
7548
7549 ins_cost(VOLATILE_REF_COST);
7550 format %{ "stlrw $src, $mem\t# int" %}
7551
7552 ins_encode(aarch64_enc_stlrw(src, mem));
7553
7554 ins_pipe(pipe_class_memory);
7555 %}
7556
7557 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7558 %{
7559 match(Set mem(StoreI mem zero));
7560
7561 ins_cost(VOLATILE_REF_COST);
7562 format %{ "stlrw zr, $mem\t# int" %}
7563
7564 ins_encode(aarch64_enc_stlrw0(mem));
7565
7566 ins_pipe(pipe_class_memory);
7567 %}
7568
7569 // Store Long (64 bit signed)
7570 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7571 %{
7572 match(Set mem (StoreL mem src));
7573
7574 ins_cost(VOLATILE_REF_COST);
7575 format %{ "stlr $src, $mem\t# int" %}
7576
7577 ins_encode(aarch64_enc_stlr(src, mem));
7578
7579 ins_pipe(pipe_class_memory);
7580 %}
7581
7582 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7583 %{
7584 match(Set mem (StoreL mem zero));
7585
7586 ins_cost(VOLATILE_REF_COST);
7587 format %{ "stlr zr, $mem\t# int" %}
7588
7589 ins_encode(aarch64_enc_stlr0(mem));
7590
7591 ins_pipe(pipe_class_memory);
7592 %}
7593
7594 // Store Pointer
7595 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7596 %{
7597 match(Set mem (StoreP mem src));
7598 predicate(n->as_Store()->barrier_data() == 0);
7599
7600 ins_cost(VOLATILE_REF_COST);
7601 format %{ "stlr $src, $mem\t# ptr" %}
7602
7603 ins_encode(aarch64_enc_stlr(src, mem));
7604
7605 ins_pipe(pipe_class_memory);
7606 %}
7607
7608 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7609 %{
7610 match(Set mem (StoreP mem zero));
7611 predicate(n->as_Store()->barrier_data() == 0);
7612
7613 ins_cost(VOLATILE_REF_COST);
7614 format %{ "stlr zr, $mem\t# ptr" %}
7615
7616 ins_encode(aarch64_enc_stlr0(mem));
7617
7618 ins_pipe(pipe_class_memory);
7619 %}
7620
7621 // Store Compressed Pointer
7622 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7623 %{
7624 match(Set mem (StoreN mem src));
7625 predicate(n->as_Store()->barrier_data() == 0);
7626
7627 ins_cost(VOLATILE_REF_COST);
7628 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7629
7630 ins_encode(aarch64_enc_stlrw(src, mem));
7631
7632 ins_pipe(pipe_class_memory);
7633 %}
7634
7635 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7636 %{
7637 match(Set mem (StoreN mem zero));
7638 predicate(n->as_Store()->barrier_data() == 0);
7639
7640 ins_cost(VOLATILE_REF_COST);
7641 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7642
7643 ins_encode(aarch64_enc_stlrw0(mem));
7644
7645 ins_pipe(pipe_class_memory);
7646 %}
7647
7648 // Store Float
7649 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7650 %{
7651 match(Set mem (StoreF mem src));
7652
7653 ins_cost(VOLATILE_REF_COST);
7654 format %{ "stlrs $src, $mem\t# float" %}
7655
7656 ins_encode( aarch64_enc_fstlrs(src, mem) );
7657
7658 ins_pipe(pipe_class_memory);
7659 %}
7660
7661 // TODO
7662 // implement storeImmF0 and storeFImmPacked
7663
7664 // Store Double
7665 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7666 %{
7667 match(Set mem (StoreD mem src));
7668
7669 ins_cost(VOLATILE_REF_COST);
7670 format %{ "stlrd $src, $mem\t# double" %}
7671
7672 ins_encode( aarch64_enc_fstlrd(src, mem) );
7673
7674 ins_pipe(pipe_class_memory);
7675 %}
7676
7677 // ---------------- end of volatile loads and stores ----------------
7678
7679 instruct cacheWB(indirect addr)
7680 %{
7681 predicate(VM_Version::supports_data_cache_line_flush());
7682 match(CacheWB addr);
7683
7684 ins_cost(100);
7685 format %{"cache wb $addr" %}
7686 ins_encode %{
7687 assert($addr->index_position() < 0, "should be");
7688 assert($addr$$disp == 0, "should be");
7689 __ cache_wb(Address($addr$$base$$Register, 0));
7690 %}
7691 ins_pipe(pipe_slow); // XXX
7692 %}
7693
7694 instruct cacheWBPreSync()
7695 %{
7696 predicate(VM_Version::supports_data_cache_line_flush());
7697 match(CacheWBPreSync);
7698
7699 ins_cost(100);
7700 format %{"cache wb presync" %}
7701 ins_encode %{
7702 __ cache_wbsync(true);
7703 %}
7704 ins_pipe(pipe_slow); // XXX
7705 %}
7706
7707 instruct cacheWBPostSync()
7708 %{
7709 predicate(VM_Version::supports_data_cache_line_flush());
7710 match(CacheWBPostSync);
7711
7712 ins_cost(100);
7713 format %{"cache wb postsync" %}
7714 ins_encode %{
7715 __ cache_wbsync(false);
7716 %}
7717 ins_pipe(pipe_slow); // XXX
7718 %}
7719
7720 // ============================================================================
7721 // BSWAP Instructions
7722
7723 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7724 match(Set dst (ReverseBytesI src));
7725
7726 ins_cost(INSN_COST);
7727 format %{ "revw $dst, $src" %}
7728
7729 ins_encode %{
7730 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7731 %}
7732
7733 ins_pipe(ialu_reg);
7734 %}
7735
7736 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7737 match(Set dst (ReverseBytesL src));
7738
7739 ins_cost(INSN_COST);
7740 format %{ "rev $dst, $src" %}
7741
7742 ins_encode %{
7743 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7744 %}
7745
7746 ins_pipe(ialu_reg);
7747 %}
7748
7749 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7750 match(Set dst (ReverseBytesUS src));
7751
7752 ins_cost(INSN_COST);
7753 format %{ "rev16w $dst, $src" %}
7754
7755 ins_encode %{
7756 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7757 %}
7758
7759 ins_pipe(ialu_reg);
7760 %}
7761
7762 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7763 match(Set dst (ReverseBytesS src));
7764
7765 ins_cost(INSN_COST);
7766 format %{ "rev16w $dst, $src\n\t"
7767 "sbfmw $dst, $dst, #0, #15" %}
7768
7769 ins_encode %{
7770 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7771 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7772 %}
7773
7774 ins_pipe(ialu_reg);
7775 %}
7776
7777 // ============================================================================
7778 // Zero Count Instructions
7779
7780 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7781 match(Set dst (CountLeadingZerosI src));
7782
7783 ins_cost(INSN_COST);
7784 format %{ "clzw $dst, $src" %}
7785 ins_encode %{
7786 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7787 %}
7788
7789 ins_pipe(ialu_reg);
7790 %}
7791
7792 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7793 match(Set dst (CountLeadingZerosL src));
7794
7795 ins_cost(INSN_COST);
7796 format %{ "clz $dst, $src" %}
7797 ins_encode %{
7798 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7799 %}
7800
7801 ins_pipe(ialu_reg);
7802 %}
7803
7804 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7805 match(Set dst (CountTrailingZerosI src));
7806
7807 ins_cost(INSN_COST * 2);
7808 format %{ "rbitw $dst, $src\n\t"
7809 "clzw $dst, $dst" %}
7810 ins_encode %{
7811 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7812 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7813 %}
7814
7815 ins_pipe(ialu_reg);
7816 %}
7817
7818 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7819 match(Set dst (CountTrailingZerosL src));
7820
7821 ins_cost(INSN_COST * 2);
7822 format %{ "rbit $dst, $src\n\t"
7823 "clz $dst, $dst" %}
7824 ins_encode %{
7825 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7826 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7827 %}
7828
7829 ins_pipe(ialu_reg);
7830 %}
7831
7832 //---------- Population Count Instructions -------------------------------------
7833 //
7834
7835 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7836 match(Set dst (PopCountI src));
7837 effect(TEMP tmp);
7838 ins_cost(INSN_COST * 13);
7839
7840 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7841 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7842 "addv $tmp, $tmp\t# vector (8B)\n\t"
7843 "mov $dst, $tmp\t# vector (1D)" %}
7844 ins_encode %{
7845 __ fmovs($tmp$$FloatRegister, $src$$Register);
7846 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7847 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7848 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7849 %}
7850
7851 ins_pipe(pipe_class_default);
7852 %}
7853
7854 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7855 match(Set dst (PopCountI (LoadI mem)));
7856 effect(TEMP tmp);
7857 ins_cost(INSN_COST * 13);
7858
7859 format %{ "ldrs $tmp, $mem\n\t"
7860 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7861 "addv $tmp, $tmp\t# vector (8B)\n\t"
7862 "mov $dst, $tmp\t# vector (1D)" %}
7863 ins_encode %{
7864 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7865 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7866 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7867 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7868 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7869 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7870 %}
7871
7872 ins_pipe(pipe_class_default);
7873 %}
7874
7875 // Note: Long.bitCount(long) returns an int.
7876 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7877 match(Set dst (PopCountL src));
7878 effect(TEMP tmp);
7879 ins_cost(INSN_COST * 13);
7880
7881 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7882 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7883 "addv $tmp, $tmp\t# vector (8B)\n\t"
7884 "mov $dst, $tmp\t# vector (1D)" %}
7885 ins_encode %{
7886 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7887 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7888 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7889 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7890 %}
7891
7892 ins_pipe(pipe_class_default);
7893 %}
7894
7895 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7896 match(Set dst (PopCountL (LoadL mem)));
7897 effect(TEMP tmp);
7898 ins_cost(INSN_COST * 13);
7899
7900 format %{ "ldrd $tmp, $mem\n\t"
7901 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7902 "addv $tmp, $tmp\t# vector (8B)\n\t"
7903 "mov $dst, $tmp\t# vector (1D)" %}
7904 ins_encode %{
7905 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7906 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7907 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7908 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7909 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7910 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7911 %}
7912
7913 ins_pipe(pipe_class_default);
7914 %}
7915
7916 // ============================================================================
7917 // VerifyVectorAlignment Instruction
7918
7919 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7920 match(Set addr (VerifyVectorAlignment addr mask));
7921 effect(KILL cr);
7922 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7923 ins_encode %{
7924 Label Lskip;
7925 // check if masked bits of addr are zero
7926 __ tst($addr$$Register, $mask$$constant);
7927 __ br(Assembler::EQ, Lskip);
7928 __ stop("verify_vector_alignment found a misaligned vector memory access");
7929 __ bind(Lskip);
7930 %}
7931 ins_pipe(pipe_slow);
7932 %}
7933
7934 // ============================================================================
7935 // MemBar Instruction
7936
7937 instruct load_fence() %{
7938 match(LoadFence);
7939 ins_cost(VOLATILE_REF_COST);
7940
7941 format %{ "load_fence" %}
7942
7943 ins_encode %{
7944 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7945 %}
7946 ins_pipe(pipe_serial);
7947 %}
7948
7949 instruct unnecessary_membar_acquire() %{
7950 predicate(unnecessary_acquire(n));
7951 match(MemBarAcquire);
7952 ins_cost(0);
7953
7954 format %{ "membar_acquire (elided)" %}
7955
7956 ins_encode %{
7957 __ block_comment("membar_acquire (elided)");
7958 %}
7959
7960 ins_pipe(pipe_class_empty);
7961 %}
7962
7963 instruct membar_acquire() %{
7964 match(MemBarAcquire);
7965 ins_cost(VOLATILE_REF_COST);
7966
7967 format %{ "membar_acquire\n\t"
7968 "dmb ishld" %}
7969
7970 ins_encode %{
7971 __ block_comment("membar_acquire");
7972 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7973 %}
7974
7975 ins_pipe(pipe_serial);
7976 %}
7977
7978
7979 instruct membar_acquire_lock() %{
7980 match(MemBarAcquireLock);
7981 ins_cost(VOLATILE_REF_COST);
7982
7983 format %{ "membar_acquire_lock (elided)" %}
7984
7985 ins_encode %{
7986 __ block_comment("membar_acquire_lock (elided)");
7987 %}
7988
7989 ins_pipe(pipe_serial);
7990 %}
7991
7992 instruct store_fence() %{
7993 match(StoreFence);
7994 ins_cost(VOLATILE_REF_COST);
7995
7996 format %{ "store_fence" %}
7997
7998 ins_encode %{
7999 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8000 %}
8001 ins_pipe(pipe_serial);
8002 %}
8003
8004 instruct unnecessary_membar_release() %{
8005 predicate(unnecessary_release(n));
8006 match(MemBarRelease);
8007 ins_cost(0);
8008
8009 format %{ "membar_release (elided)" %}
8010
8011 ins_encode %{
8012 __ block_comment("membar_release (elided)");
8013 %}
8014 ins_pipe(pipe_serial);
8015 %}
8016
8017 instruct membar_release() %{
8018 match(MemBarRelease);
8019 ins_cost(VOLATILE_REF_COST);
8020
8021 format %{ "membar_release\n\t"
8022 "dmb ishst\n\tdmb ishld" %}
8023
8024 ins_encode %{
8025 __ block_comment("membar_release");
8026 // These will be merged if AlwaysMergeDMB is enabled.
8027 __ membar(Assembler::StoreStore);
8028 __ membar(Assembler::LoadStore);
8029 %}
8030 ins_pipe(pipe_serial);
8031 %}
8032
8033 instruct membar_storestore() %{
8034 match(MemBarStoreStore);
8035 match(StoreStoreFence);
8036 ins_cost(VOLATILE_REF_COST);
8037
8038 format %{ "MEMBAR-store-store" %}
8039
8040 ins_encode %{
8041 __ membar(Assembler::StoreStore);
8042 %}
8043 ins_pipe(pipe_serial);
8044 %}
8045
8046 instruct membar_release_lock() %{
8047 match(MemBarReleaseLock);
8048 ins_cost(VOLATILE_REF_COST);
8049
8050 format %{ "membar_release_lock (elided)" %}
8051
8052 ins_encode %{
8053 __ block_comment("membar_release_lock (elided)");
8054 %}
8055
8056 ins_pipe(pipe_serial);
8057 %}
8058
8059 instruct unnecessary_membar_volatile() %{
8060 predicate(unnecessary_volatile(n));
8061 match(MemBarVolatile);
8062 ins_cost(0);
8063
8064 format %{ "membar_volatile (elided)" %}
8065
8066 ins_encode %{
8067 __ block_comment("membar_volatile (elided)");
8068 %}
8069
8070 ins_pipe(pipe_serial);
8071 %}
8072
8073 instruct membar_volatile() %{
8074 match(MemBarVolatile);
8075 ins_cost(VOLATILE_REF_COST*100);
8076
8077 format %{ "membar_volatile\n\t"
8078 "dmb ish"%}
8079
8080 ins_encode %{
8081 __ block_comment("membar_volatile");
8082 __ membar(Assembler::StoreLoad);
8083 %}
8084
8085 ins_pipe(pipe_serial);
8086 %}
8087
8088 // ============================================================================
8089 // Cast/Convert Instructions
8090
8091 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8092 match(Set dst (CastX2P src));
8093
8094 ins_cost(INSN_COST);
8095 format %{ "mov $dst, $src\t# long -> ptr" %}
8096
8097 ins_encode %{
8098 if ($dst$$reg != $src$$reg) {
8099 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8100 }
8101 %}
8102
8103 ins_pipe(ialu_reg);
8104 %}
8105
8106 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8107 match(Set dst (CastI2N src));
8108
8109 ins_cost(INSN_COST);
8110 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8111
8112 ins_encode %{
8113 if ($dst$$reg != $src$$reg) {
8114 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8115 }
8116 %}
8117
8118 ins_pipe(ialu_reg);
8119 %}
8120
8121 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8122 match(Set dst (CastP2X src));
8123
8124 ins_cost(INSN_COST);
8125 format %{ "mov $dst, $src\t# ptr -> long" %}
8126
8127 ins_encode %{
8128 if ($dst$$reg != $src$$reg) {
8129 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8130 }
8131 %}
8132
8133 ins_pipe(ialu_reg);
8134 %}
8135
8136 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8137 match(Set dst (CastP2X src));
8138
8139 ins_cost(INSN_COST);
8140 format %{ "mov $dst, $src\t# ptr -> long" %}
8141
8142 ins_encode %{
8143 if ($dst$$reg != $src$$reg) {
8144 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8145 }
8146 %}
8147
8148 ins_pipe(ialu_reg);
8149 %}
8150
8151 // Convert oop into int for vectors alignment masking
8152 instruct convP2I(iRegINoSp dst, iRegP src) %{
8153 match(Set dst (ConvL2I (CastP2X src)));
8154
8155 ins_cost(INSN_COST);
8156 format %{ "movw $dst, $src\t# ptr -> int" %}
8157 ins_encode %{
8158 __ movw($dst$$Register, $src$$Register);
8159 %}
8160
8161 ins_pipe(ialu_reg);
8162 %}
8163
8164 // Convert compressed oop into int for vectors alignment masking
8165 // in case of 32bit oops (heap < 4Gb).
8166 instruct convN2I(iRegINoSp dst, iRegN src)
8167 %{
8168 predicate(CompressedOops::shift() == 0);
8169 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8170
8171 ins_cost(INSN_COST);
8172 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8173 ins_encode %{
8174 __ movw($dst$$Register, $src$$Register);
8175 %}
8176
8177 ins_pipe(ialu_reg);
8178 %}
8179
8180
8181 // Convert oop pointer into compressed form
8182 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8183 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8184 match(Set dst (EncodeP src));
8185 effect(KILL cr);
8186 ins_cost(INSN_COST * 3);
8187 format %{ "encode_heap_oop $dst, $src" %}
8188 ins_encode %{
8189 Register s = $src$$Register;
8190 Register d = $dst$$Register;
8191 __ encode_heap_oop(d, s);
8192 %}
8193 ins_pipe(ialu_reg);
8194 %}
8195
8196 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8197 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8198 match(Set dst (EncodeP src));
8199 ins_cost(INSN_COST * 3);
8200 format %{ "encode_heap_oop_not_null $dst, $src" %}
8201 ins_encode %{
8202 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8203 %}
8204 ins_pipe(ialu_reg);
8205 %}
8206
8207 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8208 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8209 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8210 match(Set dst (DecodeN src));
8211 ins_cost(INSN_COST * 3);
8212 format %{ "decode_heap_oop $dst, $src" %}
8213 ins_encode %{
8214 Register s = $src$$Register;
8215 Register d = $dst$$Register;
8216 __ decode_heap_oop(d, s);
8217 %}
8218 ins_pipe(ialu_reg);
8219 %}
8220
8221 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8222 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8223 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8224 match(Set dst (DecodeN src));
8225 ins_cost(INSN_COST * 3);
8226 format %{ "decode_heap_oop_not_null $dst, $src" %}
8227 ins_encode %{
8228 Register s = $src$$Register;
8229 Register d = $dst$$Register;
8230 __ decode_heap_oop_not_null(d, s);
8231 %}
8232 ins_pipe(ialu_reg);
8233 %}
8234
8235 // n.b. AArch64 implementations of encode_klass_not_null and
8236 // decode_klass_not_null do not modify the flags register so, unlike
8237 // Intel, we don't kill CR as a side effect here
8238
8239 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8240 match(Set dst (EncodePKlass src));
8241
8242 ins_cost(INSN_COST * 3);
8243 format %{ "encode_klass_not_null $dst,$src" %}
8244
8245 ins_encode %{
8246 Register src_reg = as_Register($src$$reg);
8247 Register dst_reg = as_Register($dst$$reg);
8248 __ encode_klass_not_null(dst_reg, src_reg);
8249 %}
8250
8251 ins_pipe(ialu_reg);
8252 %}
8253
8254 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8255 match(Set dst (DecodeNKlass src));
8256
8257 ins_cost(INSN_COST * 3);
8258 format %{ "decode_klass_not_null $dst,$src" %}
8259
8260 ins_encode %{
8261 Register src_reg = as_Register($src$$reg);
8262 Register dst_reg = as_Register($dst$$reg);
8263 if (dst_reg != src_reg) {
8264 __ decode_klass_not_null(dst_reg, src_reg);
8265 } else {
8266 __ decode_klass_not_null(dst_reg);
8267 }
8268 %}
8269
8270 ins_pipe(ialu_reg);
8271 %}
8272
8273 instruct checkCastPP(iRegPNoSp dst)
8274 %{
8275 match(Set dst (CheckCastPP dst));
8276
8277 size(0);
8278 format %{ "# checkcastPP of $dst" %}
8279 ins_encode(/* empty encoding */);
8280 ins_pipe(pipe_class_empty);
8281 %}
8282
8283 instruct castPP(iRegPNoSp dst)
8284 %{
8285 match(Set dst (CastPP dst));
8286
8287 size(0);
8288 format %{ "# castPP of $dst" %}
8289 ins_encode(/* empty encoding */);
8290 ins_pipe(pipe_class_empty);
8291 %}
8292
8293 instruct castII(iRegI dst)
8294 %{
8295 predicate(VerifyConstraintCasts == 0);
8296 match(Set dst (CastII dst));
8297
8298 size(0);
8299 format %{ "# castII of $dst" %}
8300 ins_encode(/* empty encoding */);
8301 ins_cost(0);
8302 ins_pipe(pipe_class_empty);
8303 %}
8304
8305 instruct castII_checked(iRegI dst, rFlagsReg cr)
8306 %{
8307 predicate(VerifyConstraintCasts > 0);
8308 match(Set dst (CastII dst));
8309 effect(KILL cr);
8310
8311 format %{ "# castII_checked of $dst" %}
8312 ins_encode %{
8313 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8314 %}
8315 ins_pipe(pipe_slow);
8316 %}
8317
8318 instruct castLL(iRegL dst)
8319 %{
8320 predicate(VerifyConstraintCasts == 0);
8321 match(Set dst (CastLL dst));
8322
8323 size(0);
8324 format %{ "# castLL of $dst" %}
8325 ins_encode(/* empty encoding */);
8326 ins_cost(0);
8327 ins_pipe(pipe_class_empty);
8328 %}
8329
8330 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8331 %{
8332 predicate(VerifyConstraintCasts > 0);
8333 match(Set dst (CastLL dst));
8334 effect(KILL cr);
8335
8336 format %{ "# castLL_checked of $dst" %}
8337 ins_encode %{
8338 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8339 %}
8340 ins_pipe(pipe_slow);
8341 %}
8342
8343 instruct castHH(vRegF dst)
8344 %{
8345 match(Set dst (CastHH dst));
8346 size(0);
8347 format %{ "# castHH of $dst" %}
8348 ins_encode(/* empty encoding */);
8349 ins_cost(0);
8350 ins_pipe(pipe_class_empty);
8351 %}
8352
8353 instruct castFF(vRegF dst)
8354 %{
8355 match(Set dst (CastFF dst));
8356
8357 size(0);
8358 format %{ "# castFF of $dst" %}
8359 ins_encode(/* empty encoding */);
8360 ins_cost(0);
8361 ins_pipe(pipe_class_empty);
8362 %}
8363
8364 instruct castDD(vRegD dst)
8365 %{
8366 match(Set dst (CastDD dst));
8367
8368 size(0);
8369 format %{ "# castDD of $dst" %}
8370 ins_encode(/* empty encoding */);
8371 ins_cost(0);
8372 ins_pipe(pipe_class_empty);
8373 %}
8374
8375 instruct castVV(vReg dst)
8376 %{
8377 match(Set dst (CastVV dst));
8378
8379 size(0);
8380 format %{ "# castVV of $dst" %}
8381 ins_encode(/* empty encoding */);
8382 ins_cost(0);
8383 ins_pipe(pipe_class_empty);
8384 %}
8385
8386 instruct castVVMask(pRegGov dst)
8387 %{
8388 match(Set dst (CastVV dst));
8389
8390 size(0);
8391 format %{ "# castVV of $dst" %}
8392 ins_encode(/* empty encoding */);
8393 ins_cost(0);
8394 ins_pipe(pipe_class_empty);
8395 %}
8396
8397 // Manifest a CmpU result in an integer register.
8398 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8399 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8400 %{
8401 match(Set dst (CmpU3 src1 src2));
8402 effect(KILL flags);
8403
8404 ins_cost(INSN_COST * 3);
8405 format %{
8406 "cmpw $src1, $src2\n\t"
8407 "csetw $dst, ne\n\t"
8408 "cnegw $dst, lo\t# CmpU3(reg)"
8409 %}
8410 ins_encode %{
8411 __ cmpw($src1$$Register, $src2$$Register);
8412 __ csetw($dst$$Register, Assembler::NE);
8413 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8414 %}
8415
8416 ins_pipe(pipe_class_default);
8417 %}
8418
8419 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8420 %{
8421 match(Set dst (CmpU3 src1 src2));
8422 effect(KILL flags);
8423
8424 ins_cost(INSN_COST * 3);
8425 format %{
8426 "subsw zr, $src1, $src2\n\t"
8427 "csetw $dst, ne\n\t"
8428 "cnegw $dst, lo\t# CmpU3(imm)"
8429 %}
8430 ins_encode %{
8431 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8432 __ csetw($dst$$Register, Assembler::NE);
8433 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8434 %}
8435
8436 ins_pipe(pipe_class_default);
8437 %}
8438
8439 // Manifest a CmpUL result in an integer register.
8440 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8441 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8442 %{
8443 match(Set dst (CmpUL3 src1 src2));
8444 effect(KILL flags);
8445
8446 ins_cost(INSN_COST * 3);
8447 format %{
8448 "cmp $src1, $src2\n\t"
8449 "csetw $dst, ne\n\t"
8450 "cnegw $dst, lo\t# CmpUL3(reg)"
8451 %}
8452 ins_encode %{
8453 __ cmp($src1$$Register, $src2$$Register);
8454 __ csetw($dst$$Register, Assembler::NE);
8455 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8456 %}
8457
8458 ins_pipe(pipe_class_default);
8459 %}
8460
8461 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8462 %{
8463 match(Set dst (CmpUL3 src1 src2));
8464 effect(KILL flags);
8465
8466 ins_cost(INSN_COST * 3);
8467 format %{
8468 "subs zr, $src1, $src2\n\t"
8469 "csetw $dst, ne\n\t"
8470 "cnegw $dst, lo\t# CmpUL3(imm)"
8471 %}
8472 ins_encode %{
8473 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8474 __ csetw($dst$$Register, Assembler::NE);
8475 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8476 %}
8477
8478 ins_pipe(pipe_class_default);
8479 %}
8480
8481 // Manifest a CmpL result in an integer register.
8482 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8483 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8484 %{
8485 match(Set dst (CmpL3 src1 src2));
8486 effect(KILL flags);
8487
8488 ins_cost(INSN_COST * 3);
8489 format %{
8490 "cmp $src1, $src2\n\t"
8491 "csetw $dst, ne\n\t"
8492 "cnegw $dst, lt\t# CmpL3(reg)"
8493 %}
8494 ins_encode %{
8495 __ cmp($src1$$Register, $src2$$Register);
8496 __ csetw($dst$$Register, Assembler::NE);
8497 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8498 %}
8499
8500 ins_pipe(pipe_class_default);
8501 %}
8502
8503 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8504 %{
8505 match(Set dst (CmpL3 src1 src2));
8506 effect(KILL flags);
8507
8508 ins_cost(INSN_COST * 3);
8509 format %{
8510 "subs zr, $src1, $src2\n\t"
8511 "csetw $dst, ne\n\t"
8512 "cnegw $dst, lt\t# CmpL3(imm)"
8513 %}
8514 ins_encode %{
8515 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8516 __ csetw($dst$$Register, Assembler::NE);
8517 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8518 %}
8519
8520 ins_pipe(pipe_class_default);
8521 %}
8522
8523 // ============================================================================
8524 // Conditional Move Instructions
8525
8526 // n.b. we have identical rules for both a signed compare op (cmpOp)
8527 // and an unsigned compare op (cmpOpU). it would be nice if we could
8528 // define an op class which merged both inputs and use it to type the
8529 // argument to a single rule. unfortunatelyt his fails because the
8530 // opclass does not live up to the COND_INTER interface of its
8531 // component operands. When the generic code tries to negate the
8532 // operand it ends up running the generci Machoper::negate method
8533 // which throws a ShouldNotHappen. So, we have to provide two flavours
8534 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8535
8536 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8537 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8538
8539 ins_cost(INSN_COST * 2);
8540 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8541
8542 ins_encode %{
8543 __ cselw(as_Register($dst$$reg),
8544 as_Register($src2$$reg),
8545 as_Register($src1$$reg),
8546 (Assembler::Condition)$cmp$$cmpcode);
8547 %}
8548
8549 ins_pipe(icond_reg_reg);
8550 %}
8551
8552 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8553 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8554
8555 ins_cost(INSN_COST * 2);
8556 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8557
8558 ins_encode %{
8559 __ cselw(as_Register($dst$$reg),
8560 as_Register($src2$$reg),
8561 as_Register($src1$$reg),
8562 (Assembler::Condition)$cmp$$cmpcode);
8563 %}
8564
8565 ins_pipe(icond_reg_reg);
8566 %}
8567
8568 // special cases where one arg is zero
8569
8570 // n.b. this is selected in preference to the rule above because it
8571 // avoids loading constant 0 into a source register
8572
8573 // TODO
8574 // we ought only to be able to cull one of these variants as the ideal
8575 // transforms ought always to order the zero consistently (to left/right?)
8576
8577 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8578 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8579
8580 ins_cost(INSN_COST * 2);
8581 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8582
8583 ins_encode %{
8584 __ cselw(as_Register($dst$$reg),
8585 as_Register($src$$reg),
8586 zr,
8587 (Assembler::Condition)$cmp$$cmpcode);
8588 %}
8589
8590 ins_pipe(icond_reg);
8591 %}
8592
8593 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8594 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8595
8596 ins_cost(INSN_COST * 2);
8597 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8598
8599 ins_encode %{
8600 __ cselw(as_Register($dst$$reg),
8601 as_Register($src$$reg),
8602 zr,
8603 (Assembler::Condition)$cmp$$cmpcode);
8604 %}
8605
8606 ins_pipe(icond_reg);
8607 %}
8608
8609 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8610 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8611
8612 ins_cost(INSN_COST * 2);
8613 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8614
8615 ins_encode %{
8616 __ cselw(as_Register($dst$$reg),
8617 zr,
8618 as_Register($src$$reg),
8619 (Assembler::Condition)$cmp$$cmpcode);
8620 %}
8621
8622 ins_pipe(icond_reg);
8623 %}
8624
8625 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8626 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8627
8628 ins_cost(INSN_COST * 2);
8629 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8630
8631 ins_encode %{
8632 __ cselw(as_Register($dst$$reg),
8633 zr,
8634 as_Register($src$$reg),
8635 (Assembler::Condition)$cmp$$cmpcode);
8636 %}
8637
8638 ins_pipe(icond_reg);
8639 %}
8640
8641 // special case for creating a boolean 0 or 1
8642
8643 // n.b. this is selected in preference to the rule above because it
8644 // avoids loading constants 0 and 1 into a source register
8645
8646 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8647 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8648
8649 ins_cost(INSN_COST * 2);
8650 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8651
8652 ins_encode %{
8653 // equivalently
8654 // cset(as_Register($dst$$reg),
8655 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8656 __ csincw(as_Register($dst$$reg),
8657 zr,
8658 zr,
8659 (Assembler::Condition)$cmp$$cmpcode);
8660 %}
8661
8662 ins_pipe(icond_none);
8663 %}
8664
8665 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8666 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8667
8668 ins_cost(INSN_COST * 2);
8669 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8670
8671 ins_encode %{
8672 // equivalently
8673 // cset(as_Register($dst$$reg),
8674 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8675 __ csincw(as_Register($dst$$reg),
8676 zr,
8677 zr,
8678 (Assembler::Condition)$cmp$$cmpcode);
8679 %}
8680
8681 ins_pipe(icond_none);
8682 %}
8683
8684 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8685 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8686
8687 ins_cost(INSN_COST * 2);
8688 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8689
8690 ins_encode %{
8691 __ csel(as_Register($dst$$reg),
8692 as_Register($src2$$reg),
8693 as_Register($src1$$reg),
8694 (Assembler::Condition)$cmp$$cmpcode);
8695 %}
8696
8697 ins_pipe(icond_reg_reg);
8698 %}
8699
8700 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8701 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8702
8703 ins_cost(INSN_COST * 2);
8704 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8705
8706 ins_encode %{
8707 __ csel(as_Register($dst$$reg),
8708 as_Register($src2$$reg),
8709 as_Register($src1$$reg),
8710 (Assembler::Condition)$cmp$$cmpcode);
8711 %}
8712
8713 ins_pipe(icond_reg_reg);
8714 %}
8715
8716 // special cases where one arg is zero
8717
8718 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8719 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8720
8721 ins_cost(INSN_COST * 2);
8722 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8723
8724 ins_encode %{
8725 __ csel(as_Register($dst$$reg),
8726 zr,
8727 as_Register($src$$reg),
8728 (Assembler::Condition)$cmp$$cmpcode);
8729 %}
8730
8731 ins_pipe(icond_reg);
8732 %}
8733
8734 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8735 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8736
8737 ins_cost(INSN_COST * 2);
8738 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8739
8740 ins_encode %{
8741 __ csel(as_Register($dst$$reg),
8742 zr,
8743 as_Register($src$$reg),
8744 (Assembler::Condition)$cmp$$cmpcode);
8745 %}
8746
8747 ins_pipe(icond_reg);
8748 %}
8749
8750 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8751 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8752
8753 ins_cost(INSN_COST * 2);
8754 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8755
8756 ins_encode %{
8757 __ csel(as_Register($dst$$reg),
8758 as_Register($src$$reg),
8759 zr,
8760 (Assembler::Condition)$cmp$$cmpcode);
8761 %}
8762
8763 ins_pipe(icond_reg);
8764 %}
8765
8766 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8767 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8768
8769 ins_cost(INSN_COST * 2);
8770 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8771
8772 ins_encode %{
8773 __ csel(as_Register($dst$$reg),
8774 as_Register($src$$reg),
8775 zr,
8776 (Assembler::Condition)$cmp$$cmpcode);
8777 %}
8778
8779 ins_pipe(icond_reg);
8780 %}
8781
8782 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8783 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8784
8785 ins_cost(INSN_COST * 2);
8786 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8787
8788 ins_encode %{
8789 __ csel(as_Register($dst$$reg),
8790 as_Register($src2$$reg),
8791 as_Register($src1$$reg),
8792 (Assembler::Condition)$cmp$$cmpcode);
8793 %}
8794
8795 ins_pipe(icond_reg_reg);
8796 %}
8797
8798 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8799 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8800
8801 ins_cost(INSN_COST * 2);
8802 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8803
8804 ins_encode %{
8805 __ csel(as_Register($dst$$reg),
8806 as_Register($src2$$reg),
8807 as_Register($src1$$reg),
8808 (Assembler::Condition)$cmp$$cmpcode);
8809 %}
8810
8811 ins_pipe(icond_reg_reg);
8812 %}
8813
8814 // special cases where one arg is zero
8815
8816 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8817 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8818
8819 ins_cost(INSN_COST * 2);
8820 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8821
8822 ins_encode %{
8823 __ csel(as_Register($dst$$reg),
8824 zr,
8825 as_Register($src$$reg),
8826 (Assembler::Condition)$cmp$$cmpcode);
8827 %}
8828
8829 ins_pipe(icond_reg);
8830 %}
8831
8832 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8833 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8834
8835 ins_cost(INSN_COST * 2);
8836 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8837
8838 ins_encode %{
8839 __ csel(as_Register($dst$$reg),
8840 zr,
8841 as_Register($src$$reg),
8842 (Assembler::Condition)$cmp$$cmpcode);
8843 %}
8844
8845 ins_pipe(icond_reg);
8846 %}
8847
8848 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8849 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8850
8851 ins_cost(INSN_COST * 2);
8852 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8853
8854 ins_encode %{
8855 __ csel(as_Register($dst$$reg),
8856 as_Register($src$$reg),
8857 zr,
8858 (Assembler::Condition)$cmp$$cmpcode);
8859 %}
8860
8861 ins_pipe(icond_reg);
8862 %}
8863
8864 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8865 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8866
8867 ins_cost(INSN_COST * 2);
8868 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8869
8870 ins_encode %{
8871 __ csel(as_Register($dst$$reg),
8872 as_Register($src$$reg),
8873 zr,
8874 (Assembler::Condition)$cmp$$cmpcode);
8875 %}
8876
8877 ins_pipe(icond_reg);
8878 %}
8879
8880 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8881 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8882
8883 ins_cost(INSN_COST * 2);
8884 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8885
8886 ins_encode %{
8887 __ cselw(as_Register($dst$$reg),
8888 as_Register($src2$$reg),
8889 as_Register($src1$$reg),
8890 (Assembler::Condition)$cmp$$cmpcode);
8891 %}
8892
8893 ins_pipe(icond_reg_reg);
8894 %}
8895
8896 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8897 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8898
8899 ins_cost(INSN_COST * 2);
8900 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8901
8902 ins_encode %{
8903 __ cselw(as_Register($dst$$reg),
8904 as_Register($src2$$reg),
8905 as_Register($src1$$reg),
8906 (Assembler::Condition)$cmp$$cmpcode);
8907 %}
8908
8909 ins_pipe(icond_reg_reg);
8910 %}
8911
8912 // special cases where one arg is zero
8913
8914 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8915 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8916
8917 ins_cost(INSN_COST * 2);
8918 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8919
8920 ins_encode %{
8921 __ cselw(as_Register($dst$$reg),
8922 zr,
8923 as_Register($src$$reg),
8924 (Assembler::Condition)$cmp$$cmpcode);
8925 %}
8926
8927 ins_pipe(icond_reg);
8928 %}
8929
8930 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8931 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8932
8933 ins_cost(INSN_COST * 2);
8934 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8935
8936 ins_encode %{
8937 __ cselw(as_Register($dst$$reg),
8938 zr,
8939 as_Register($src$$reg),
8940 (Assembler::Condition)$cmp$$cmpcode);
8941 %}
8942
8943 ins_pipe(icond_reg);
8944 %}
8945
8946 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8947 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8948
8949 ins_cost(INSN_COST * 2);
8950 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8951
8952 ins_encode %{
8953 __ cselw(as_Register($dst$$reg),
8954 as_Register($src$$reg),
8955 zr,
8956 (Assembler::Condition)$cmp$$cmpcode);
8957 %}
8958
8959 ins_pipe(icond_reg);
8960 %}
8961
8962 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8963 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8964
8965 ins_cost(INSN_COST * 2);
8966 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8967
8968 ins_encode %{
8969 __ cselw(as_Register($dst$$reg),
8970 as_Register($src$$reg),
8971 zr,
8972 (Assembler::Condition)$cmp$$cmpcode);
8973 %}
8974
8975 ins_pipe(icond_reg);
8976 %}
8977
8978 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8979 %{
8980 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8981
8982 ins_cost(INSN_COST * 3);
8983
8984 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8985 ins_encode %{
8986 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8987 __ fcsels(as_FloatRegister($dst$$reg),
8988 as_FloatRegister($src2$$reg),
8989 as_FloatRegister($src1$$reg),
8990 cond);
8991 %}
8992
8993 ins_pipe(fp_cond_reg_reg_s);
8994 %}
8995
8996 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8997 %{
8998 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8999
9000 ins_cost(INSN_COST * 3);
9001
9002 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9003 ins_encode %{
9004 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9005 __ fcsels(as_FloatRegister($dst$$reg),
9006 as_FloatRegister($src2$$reg),
9007 as_FloatRegister($src1$$reg),
9008 cond);
9009 %}
9010
9011 ins_pipe(fp_cond_reg_reg_s);
9012 %}
9013
9014 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9015 %{
9016 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9017
9018 ins_cost(INSN_COST * 3);
9019
9020 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9021 ins_encode %{
9022 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9023 __ fcseld(as_FloatRegister($dst$$reg),
9024 as_FloatRegister($src2$$reg),
9025 as_FloatRegister($src1$$reg),
9026 cond);
9027 %}
9028
9029 ins_pipe(fp_cond_reg_reg_d);
9030 %}
9031
9032 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9033 %{
9034 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9035
9036 ins_cost(INSN_COST * 3);
9037
9038 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9039 ins_encode %{
9040 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9041 __ fcseld(as_FloatRegister($dst$$reg),
9042 as_FloatRegister($src2$$reg),
9043 as_FloatRegister($src1$$reg),
9044 cond);
9045 %}
9046
9047 ins_pipe(fp_cond_reg_reg_d);
9048 %}
9049
9050 // ============================================================================
9051 // Arithmetic Instructions
9052 //
9053
9054 // Integer Addition
9055
9056 // TODO
9057 // these currently employ operations which do not set CR and hence are
9058 // not flagged as killing CR but we would like to isolate the cases
9059 // where we want to set flags from those where we don't. need to work
9060 // out how to do that.
9061
9062 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9063 match(Set dst (AddI src1 src2));
9064
9065 ins_cost(INSN_COST);
9066 format %{ "addw $dst, $src1, $src2" %}
9067
9068 ins_encode %{
9069 __ addw(as_Register($dst$$reg),
9070 as_Register($src1$$reg),
9071 as_Register($src2$$reg));
9072 %}
9073
9074 ins_pipe(ialu_reg_reg);
9075 %}
9076
9077 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9078 match(Set dst (AddI src1 src2));
9079
9080 ins_cost(INSN_COST);
9081 format %{ "addw $dst, $src1, $src2" %}
9082
9083 // use opcode to indicate that this is an add not a sub
9084 opcode(0x0);
9085
9086 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9087
9088 ins_pipe(ialu_reg_imm);
9089 %}
9090
9091 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9092 match(Set dst (AddI (ConvL2I src1) src2));
9093
9094 ins_cost(INSN_COST);
9095 format %{ "addw $dst, $src1, $src2" %}
9096
9097 // use opcode to indicate that this is an add not a sub
9098 opcode(0x0);
9099
9100 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9101
9102 ins_pipe(ialu_reg_imm);
9103 %}
9104
9105 // Pointer Addition
9106 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9107 match(Set dst (AddP src1 src2));
9108
9109 ins_cost(INSN_COST);
9110 format %{ "add $dst, $src1, $src2\t# ptr" %}
9111
9112 ins_encode %{
9113 __ add(as_Register($dst$$reg),
9114 as_Register($src1$$reg),
9115 as_Register($src2$$reg));
9116 %}
9117
9118 ins_pipe(ialu_reg_reg);
9119 %}
9120
9121 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9122 match(Set dst (AddP src1 (ConvI2L src2)));
9123
9124 ins_cost(1.9 * INSN_COST);
9125 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9126
9127 ins_encode %{
9128 __ add(as_Register($dst$$reg),
9129 as_Register($src1$$reg),
9130 as_Register($src2$$reg), ext::sxtw);
9131 %}
9132
9133 ins_pipe(ialu_reg_reg);
9134 %}
9135
9136 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9137 match(Set dst (AddP src1 (LShiftL src2 scale)));
9138
9139 ins_cost(1.9 * INSN_COST);
9140 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9141
9142 ins_encode %{
9143 __ lea(as_Register($dst$$reg),
9144 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9145 Address::lsl($scale$$constant)));
9146 %}
9147
9148 ins_pipe(ialu_reg_reg_shift);
9149 %}
9150
9151 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9152 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9153
9154 ins_cost(1.9 * INSN_COST);
9155 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9156
9157 ins_encode %{
9158 __ lea(as_Register($dst$$reg),
9159 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9160 Address::sxtw($scale$$constant)));
9161 %}
9162
9163 ins_pipe(ialu_reg_reg_shift);
9164 %}
9165
9166 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9167 match(Set dst (LShiftL (ConvI2L src) scale));
9168
9169 ins_cost(INSN_COST);
9170 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9171
9172 ins_encode %{
9173 __ sbfiz(as_Register($dst$$reg),
9174 as_Register($src$$reg),
9175 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9176 %}
9177
9178 ins_pipe(ialu_reg_shift);
9179 %}
9180
9181 // Pointer Immediate Addition
9182 // n.b. this needs to be more expensive than using an indirect memory
9183 // operand
9184 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9185 match(Set dst (AddP src1 src2));
9186
9187 ins_cost(INSN_COST);
9188 format %{ "add $dst, $src1, $src2\t# ptr" %}
9189
9190 // use opcode to indicate that this is an add not a sub
9191 opcode(0x0);
9192
9193 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9194
9195 ins_pipe(ialu_reg_imm);
9196 %}
9197
9198 // Long Addition
9199 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9200
9201 match(Set dst (AddL src1 src2));
9202
9203 ins_cost(INSN_COST);
9204 format %{ "add $dst, $src1, $src2" %}
9205
9206 ins_encode %{
9207 __ add(as_Register($dst$$reg),
9208 as_Register($src1$$reg),
9209 as_Register($src2$$reg));
9210 %}
9211
9212 ins_pipe(ialu_reg_reg);
9213 %}
9214
9215 // No constant pool entries requiredLong Immediate Addition.
9216 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9217 match(Set dst (AddL src1 src2));
9218
9219 ins_cost(INSN_COST);
9220 format %{ "add $dst, $src1, $src2" %}
9221
9222 // use opcode to indicate that this is an add not a sub
9223 opcode(0x0);
9224
9225 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9226
9227 ins_pipe(ialu_reg_imm);
9228 %}
9229
9230 // Integer Subtraction
9231 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9232 match(Set dst (SubI src1 src2));
9233
9234 ins_cost(INSN_COST);
9235 format %{ "subw $dst, $src1, $src2" %}
9236
9237 ins_encode %{
9238 __ subw(as_Register($dst$$reg),
9239 as_Register($src1$$reg),
9240 as_Register($src2$$reg));
9241 %}
9242
9243 ins_pipe(ialu_reg_reg);
9244 %}
9245
9246 // Immediate Subtraction
9247 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9248 match(Set dst (SubI src1 src2));
9249
9250 ins_cost(INSN_COST);
9251 format %{ "subw $dst, $src1, $src2" %}
9252
9253 // use opcode to indicate that this is a sub not an add
9254 opcode(0x1);
9255
9256 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9257
9258 ins_pipe(ialu_reg_imm);
9259 %}
9260
9261 // Long Subtraction
9262 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9263
9264 match(Set dst (SubL src1 src2));
9265
9266 ins_cost(INSN_COST);
9267 format %{ "sub $dst, $src1, $src2" %}
9268
9269 ins_encode %{
9270 __ sub(as_Register($dst$$reg),
9271 as_Register($src1$$reg),
9272 as_Register($src2$$reg));
9273 %}
9274
9275 ins_pipe(ialu_reg_reg);
9276 %}
9277
9278 // No constant pool entries requiredLong Immediate Subtraction.
9279 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9280 match(Set dst (SubL src1 src2));
9281
9282 ins_cost(INSN_COST);
9283 format %{ "sub$dst, $src1, $src2" %}
9284
9285 // use opcode to indicate that this is a sub not an add
9286 opcode(0x1);
9287
9288 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9289
9290 ins_pipe(ialu_reg_imm);
9291 %}
9292
9293 // Integer Negation (special case for sub)
9294
9295 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9296 match(Set dst (SubI zero src));
9297
9298 ins_cost(INSN_COST);
9299 format %{ "negw $dst, $src\t# int" %}
9300
9301 ins_encode %{
9302 __ negw(as_Register($dst$$reg),
9303 as_Register($src$$reg));
9304 %}
9305
9306 ins_pipe(ialu_reg);
9307 %}
9308
9309 // Long Negation
9310
9311 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9312 match(Set dst (SubL zero src));
9313
9314 ins_cost(INSN_COST);
9315 format %{ "neg $dst, $src\t# long" %}
9316
9317 ins_encode %{
9318 __ neg(as_Register($dst$$reg),
9319 as_Register($src$$reg));
9320 %}
9321
9322 ins_pipe(ialu_reg);
9323 %}
9324
9325 // Integer Multiply
9326
9327 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9328 match(Set dst (MulI src1 src2));
9329
9330 ins_cost(INSN_COST * 3);
9331 format %{ "mulw $dst, $src1, $src2" %}
9332
9333 ins_encode %{
9334 __ mulw(as_Register($dst$$reg),
9335 as_Register($src1$$reg),
9336 as_Register($src2$$reg));
9337 %}
9338
9339 ins_pipe(imul_reg_reg);
9340 %}
9341
9342 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9343 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9344
9345 ins_cost(INSN_COST * 3);
9346 format %{ "smull $dst, $src1, $src2" %}
9347
9348 ins_encode %{
9349 __ smull(as_Register($dst$$reg),
9350 as_Register($src1$$reg),
9351 as_Register($src2$$reg));
9352 %}
9353
9354 ins_pipe(imul_reg_reg);
9355 %}
9356
9357 // Long Multiply
9358
9359 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9360 match(Set dst (MulL src1 src2));
9361
9362 ins_cost(INSN_COST * 5);
9363 format %{ "mul $dst, $src1, $src2" %}
9364
9365 ins_encode %{
9366 __ mul(as_Register($dst$$reg),
9367 as_Register($src1$$reg),
9368 as_Register($src2$$reg));
9369 %}
9370
9371 ins_pipe(lmul_reg_reg);
9372 %}
9373
9374 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9375 %{
9376 match(Set dst (MulHiL src1 src2));
9377
9378 ins_cost(INSN_COST * 7);
9379 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9380
9381 ins_encode %{
9382 __ smulh(as_Register($dst$$reg),
9383 as_Register($src1$$reg),
9384 as_Register($src2$$reg));
9385 %}
9386
9387 ins_pipe(lmul_reg_reg);
9388 %}
9389
9390 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9391 %{
9392 match(Set dst (UMulHiL src1 src2));
9393
9394 ins_cost(INSN_COST * 7);
9395 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9396
9397 ins_encode %{
9398 __ umulh(as_Register($dst$$reg),
9399 as_Register($src1$$reg),
9400 as_Register($src2$$reg));
9401 %}
9402
9403 ins_pipe(lmul_reg_reg);
9404 %}
9405
9406 // Combined Integer Multiply & Add/Sub
9407
9408 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9409 match(Set dst (AddI src3 (MulI src1 src2)));
9410
9411 ins_cost(INSN_COST * 3);
9412 format %{ "madd $dst, $src1, $src2, $src3" %}
9413
9414 ins_encode %{
9415 __ maddw(as_Register($dst$$reg),
9416 as_Register($src1$$reg),
9417 as_Register($src2$$reg),
9418 as_Register($src3$$reg));
9419 %}
9420
9421 ins_pipe(imac_reg_reg);
9422 %}
9423
9424 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9425 match(Set dst (SubI src3 (MulI src1 src2)));
9426
9427 ins_cost(INSN_COST * 3);
9428 format %{ "msub $dst, $src1, $src2, $src3" %}
9429
9430 ins_encode %{
9431 __ msubw(as_Register($dst$$reg),
9432 as_Register($src1$$reg),
9433 as_Register($src2$$reg),
9434 as_Register($src3$$reg));
9435 %}
9436
9437 ins_pipe(imac_reg_reg);
9438 %}
9439
9440 // Combined Integer Multiply & Neg
9441
9442 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9443 match(Set dst (MulI (SubI zero src1) src2));
9444
9445 ins_cost(INSN_COST * 3);
9446 format %{ "mneg $dst, $src1, $src2" %}
9447
9448 ins_encode %{
9449 __ mnegw(as_Register($dst$$reg),
9450 as_Register($src1$$reg),
9451 as_Register($src2$$reg));
9452 %}
9453
9454 ins_pipe(imac_reg_reg);
9455 %}
9456
9457 // Combined Long Multiply & Add/Sub
9458
9459 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9460 match(Set dst (AddL src3 (MulL src1 src2)));
9461
9462 ins_cost(INSN_COST * 5);
9463 format %{ "madd $dst, $src1, $src2, $src3" %}
9464
9465 ins_encode %{
9466 __ madd(as_Register($dst$$reg),
9467 as_Register($src1$$reg),
9468 as_Register($src2$$reg),
9469 as_Register($src3$$reg));
9470 %}
9471
9472 ins_pipe(lmac_reg_reg);
9473 %}
9474
9475 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9476 match(Set dst (SubL src3 (MulL src1 src2)));
9477
9478 ins_cost(INSN_COST * 5);
9479 format %{ "msub $dst, $src1, $src2, $src3" %}
9480
9481 ins_encode %{
9482 __ msub(as_Register($dst$$reg),
9483 as_Register($src1$$reg),
9484 as_Register($src2$$reg),
9485 as_Register($src3$$reg));
9486 %}
9487
9488 ins_pipe(lmac_reg_reg);
9489 %}
9490
9491 // Combined Long Multiply & Neg
9492
9493 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9494 match(Set dst (MulL (SubL zero src1) src2));
9495
9496 ins_cost(INSN_COST * 5);
9497 format %{ "mneg $dst, $src1, $src2" %}
9498
9499 ins_encode %{
9500 __ mneg(as_Register($dst$$reg),
9501 as_Register($src1$$reg),
9502 as_Register($src2$$reg));
9503 %}
9504
9505 ins_pipe(lmac_reg_reg);
9506 %}
9507
9508 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9509
9510 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9511 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9512
9513 ins_cost(INSN_COST * 3);
9514 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9515
9516 ins_encode %{
9517 __ smaddl(as_Register($dst$$reg),
9518 as_Register($src1$$reg),
9519 as_Register($src2$$reg),
9520 as_Register($src3$$reg));
9521 %}
9522
9523 ins_pipe(imac_reg_reg);
9524 %}
9525
9526 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9527 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9528
9529 ins_cost(INSN_COST * 3);
9530 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9531
9532 ins_encode %{
9533 __ smsubl(as_Register($dst$$reg),
9534 as_Register($src1$$reg),
9535 as_Register($src2$$reg),
9536 as_Register($src3$$reg));
9537 %}
9538
9539 ins_pipe(imac_reg_reg);
9540 %}
9541
9542 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9543 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9544
9545 ins_cost(INSN_COST * 3);
9546 format %{ "smnegl $dst, $src1, $src2" %}
9547
9548 ins_encode %{
9549 __ smnegl(as_Register($dst$$reg),
9550 as_Register($src1$$reg),
9551 as_Register($src2$$reg));
9552 %}
9553
9554 ins_pipe(imac_reg_reg);
9555 %}
9556
9557 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9558
9559 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9560 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9561
9562 ins_cost(INSN_COST * 5);
9563 format %{ "mulw rscratch1, $src1, $src2\n\t"
9564 "maddw $dst, $src3, $src4, rscratch1" %}
9565
9566 ins_encode %{
9567 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9568 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9569
9570 ins_pipe(imac_reg_reg);
9571 %}
9572
9573 // Integer Divide
9574
9575 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9576 match(Set dst (DivI src1 src2));
9577
9578 ins_cost(INSN_COST * 19);
9579 format %{ "sdivw $dst, $src1, $src2" %}
9580
9581 ins_encode(aarch64_enc_divw(dst, src1, src2));
9582 ins_pipe(idiv_reg_reg);
9583 %}
9584
9585 // Long Divide
9586
9587 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9588 match(Set dst (DivL src1 src2));
9589
9590 ins_cost(INSN_COST * 35);
9591 format %{ "sdiv $dst, $src1, $src2" %}
9592
9593 ins_encode(aarch64_enc_div(dst, src1, src2));
9594 ins_pipe(ldiv_reg_reg);
9595 %}
9596
9597 // Integer Remainder
9598
9599 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9600 match(Set dst (ModI src1 src2));
9601
9602 ins_cost(INSN_COST * 22);
9603 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9604 "msubw $dst, rscratch1, $src2, $src1" %}
9605
9606 ins_encode(aarch64_enc_modw(dst, src1, src2));
9607 ins_pipe(idiv_reg_reg);
9608 %}
9609
9610 // Long Remainder
9611
9612 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9613 match(Set dst (ModL src1 src2));
9614
9615 ins_cost(INSN_COST * 38);
9616 format %{ "sdiv rscratch1, $src1, $src2\n"
9617 "msub $dst, rscratch1, $src2, $src1" %}
9618
9619 ins_encode(aarch64_enc_mod(dst, src1, src2));
9620 ins_pipe(ldiv_reg_reg);
9621 %}
9622
9623 // Unsigned Integer Divide
9624
9625 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9626 match(Set dst (UDivI src1 src2));
9627
9628 ins_cost(INSN_COST * 19);
9629 format %{ "udivw $dst, $src1, $src2" %}
9630
9631 ins_encode %{
9632 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9633 %}
9634
9635 ins_pipe(idiv_reg_reg);
9636 %}
9637
9638 // Unsigned Long Divide
9639
9640 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9641 match(Set dst (UDivL src1 src2));
9642
9643 ins_cost(INSN_COST * 35);
9644 format %{ "udiv $dst, $src1, $src2" %}
9645
9646 ins_encode %{
9647 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9648 %}
9649
9650 ins_pipe(ldiv_reg_reg);
9651 %}
9652
9653 // Unsigned Integer Remainder
9654
9655 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9656 match(Set dst (UModI src1 src2));
9657
9658 ins_cost(INSN_COST * 22);
9659 format %{ "udivw rscratch1, $src1, $src2\n\t"
9660 "msubw $dst, rscratch1, $src2, $src1" %}
9661
9662 ins_encode %{
9663 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9664 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9665 %}
9666
9667 ins_pipe(idiv_reg_reg);
9668 %}
9669
9670 // Unsigned Long Remainder
9671
9672 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9673 match(Set dst (UModL src1 src2));
9674
9675 ins_cost(INSN_COST * 38);
9676 format %{ "udiv rscratch1, $src1, $src2\n"
9677 "msub $dst, rscratch1, $src2, $src1" %}
9678
9679 ins_encode %{
9680 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9681 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9682 %}
9683
9684 ins_pipe(ldiv_reg_reg);
9685 %}
9686
9687 // Integer Shifts
9688
9689 // Shift Left Register
9690 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9691 match(Set dst (LShiftI src1 src2));
9692
9693 ins_cost(INSN_COST * 2);
9694 format %{ "lslvw $dst, $src1, $src2" %}
9695
9696 ins_encode %{
9697 __ lslvw(as_Register($dst$$reg),
9698 as_Register($src1$$reg),
9699 as_Register($src2$$reg));
9700 %}
9701
9702 ins_pipe(ialu_reg_reg_vshift);
9703 %}
9704
9705 // Shift Left Immediate
9706 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9707 match(Set dst (LShiftI src1 src2));
9708
9709 ins_cost(INSN_COST);
9710 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9711
9712 ins_encode %{
9713 __ lslw(as_Register($dst$$reg),
9714 as_Register($src1$$reg),
9715 $src2$$constant & 0x1f);
9716 %}
9717
9718 ins_pipe(ialu_reg_shift);
9719 %}
9720
9721 // Shift Right Logical Register
9722 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9723 match(Set dst (URShiftI src1 src2));
9724
9725 ins_cost(INSN_COST * 2);
9726 format %{ "lsrvw $dst, $src1, $src2" %}
9727
9728 ins_encode %{
9729 __ lsrvw(as_Register($dst$$reg),
9730 as_Register($src1$$reg),
9731 as_Register($src2$$reg));
9732 %}
9733
9734 ins_pipe(ialu_reg_reg_vshift);
9735 %}
9736
9737 // Shift Right Logical Immediate
9738 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9739 match(Set dst (URShiftI src1 src2));
9740
9741 ins_cost(INSN_COST);
9742 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9743
9744 ins_encode %{
9745 __ lsrw(as_Register($dst$$reg),
9746 as_Register($src1$$reg),
9747 $src2$$constant & 0x1f);
9748 %}
9749
9750 ins_pipe(ialu_reg_shift);
9751 %}
9752
9753 // Shift Right Arithmetic Register
9754 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9755 match(Set dst (RShiftI src1 src2));
9756
9757 ins_cost(INSN_COST * 2);
9758 format %{ "asrvw $dst, $src1, $src2" %}
9759
9760 ins_encode %{
9761 __ asrvw(as_Register($dst$$reg),
9762 as_Register($src1$$reg),
9763 as_Register($src2$$reg));
9764 %}
9765
9766 ins_pipe(ialu_reg_reg_vshift);
9767 %}
9768
9769 // Shift Right Arithmetic Immediate
9770 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9771 match(Set dst (RShiftI src1 src2));
9772
9773 ins_cost(INSN_COST);
9774 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9775
9776 ins_encode %{
9777 __ asrw(as_Register($dst$$reg),
9778 as_Register($src1$$reg),
9779 $src2$$constant & 0x1f);
9780 %}
9781
9782 ins_pipe(ialu_reg_shift);
9783 %}
9784
9785 // Combined Int Mask and Right Shift (using UBFM)
9786 // TODO
9787
9788 // Long Shifts
9789
9790 // Shift Left Register
9791 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9792 match(Set dst (LShiftL src1 src2));
9793
9794 ins_cost(INSN_COST * 2);
9795 format %{ "lslv $dst, $src1, $src2" %}
9796
9797 ins_encode %{
9798 __ lslv(as_Register($dst$$reg),
9799 as_Register($src1$$reg),
9800 as_Register($src2$$reg));
9801 %}
9802
9803 ins_pipe(ialu_reg_reg_vshift);
9804 %}
9805
9806 // Shift Left Immediate
9807 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9808 match(Set dst (LShiftL src1 src2));
9809
9810 ins_cost(INSN_COST);
9811 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9812
9813 ins_encode %{
9814 __ lsl(as_Register($dst$$reg),
9815 as_Register($src1$$reg),
9816 $src2$$constant & 0x3f);
9817 %}
9818
9819 ins_pipe(ialu_reg_shift);
9820 %}
9821
9822 // Shift Right Logical Register
9823 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9824 match(Set dst (URShiftL src1 src2));
9825
9826 ins_cost(INSN_COST * 2);
9827 format %{ "lsrv $dst, $src1, $src2" %}
9828
9829 ins_encode %{
9830 __ lsrv(as_Register($dst$$reg),
9831 as_Register($src1$$reg),
9832 as_Register($src2$$reg));
9833 %}
9834
9835 ins_pipe(ialu_reg_reg_vshift);
9836 %}
9837
9838 // Shift Right Logical Immediate
9839 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9840 match(Set dst (URShiftL src1 src2));
9841
9842 ins_cost(INSN_COST);
9843 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9844
9845 ins_encode %{
9846 __ lsr(as_Register($dst$$reg),
9847 as_Register($src1$$reg),
9848 $src2$$constant & 0x3f);
9849 %}
9850
9851 ins_pipe(ialu_reg_shift);
9852 %}
9853
9854 // A special-case pattern for card table stores.
9855 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9856 match(Set dst (URShiftL (CastP2X src1) src2));
9857
9858 ins_cost(INSN_COST);
9859 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9860
9861 ins_encode %{
9862 __ lsr(as_Register($dst$$reg),
9863 as_Register($src1$$reg),
9864 $src2$$constant & 0x3f);
9865 %}
9866
9867 ins_pipe(ialu_reg_shift);
9868 %}
9869
9870 // Shift Right Arithmetic Register
9871 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9872 match(Set dst (RShiftL src1 src2));
9873
9874 ins_cost(INSN_COST * 2);
9875 format %{ "asrv $dst, $src1, $src2" %}
9876
9877 ins_encode %{
9878 __ asrv(as_Register($dst$$reg),
9879 as_Register($src1$$reg),
9880 as_Register($src2$$reg));
9881 %}
9882
9883 ins_pipe(ialu_reg_reg_vshift);
9884 %}
9885
9886 // Shift Right Arithmetic Immediate
9887 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9888 match(Set dst (RShiftL src1 src2));
9889
9890 ins_cost(INSN_COST);
9891 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9892
9893 ins_encode %{
9894 __ asr(as_Register($dst$$reg),
9895 as_Register($src1$$reg),
9896 $src2$$constant & 0x3f);
9897 %}
9898
9899 ins_pipe(ialu_reg_shift);
9900 %}
9901
9902 // BEGIN This section of the file is automatically generated. Do not edit --------------
9903 // This section is generated from aarch64_ad.m4
9904
9905 // This pattern is automatically generated from aarch64_ad.m4
9906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9907 instruct regL_not_reg(iRegLNoSp dst,
9908 iRegL src1, immL_M1 m1,
9909 rFlagsReg cr) %{
9910 match(Set dst (XorL src1 m1));
9911 ins_cost(INSN_COST);
9912 format %{ "eon $dst, $src1, zr" %}
9913
9914 ins_encode %{
9915 __ eon(as_Register($dst$$reg),
9916 as_Register($src1$$reg),
9917 zr,
9918 Assembler::LSL, 0);
9919 %}
9920
9921 ins_pipe(ialu_reg);
9922 %}
9923
9924 // This pattern is automatically generated from aarch64_ad.m4
9925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9926 instruct regI_not_reg(iRegINoSp dst,
9927 iRegIorL2I src1, immI_M1 m1,
9928 rFlagsReg cr) %{
9929 match(Set dst (XorI src1 m1));
9930 ins_cost(INSN_COST);
9931 format %{ "eonw $dst, $src1, zr" %}
9932
9933 ins_encode %{
9934 __ eonw(as_Register($dst$$reg),
9935 as_Register($src1$$reg),
9936 zr,
9937 Assembler::LSL, 0);
9938 %}
9939
9940 ins_pipe(ialu_reg);
9941 %}
9942
9943 // This pattern is automatically generated from aarch64_ad.m4
9944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9945 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9946 immI0 zero, iRegIorL2I src1, immI src2) %{
9947 match(Set dst (SubI zero (URShiftI src1 src2)));
9948
9949 ins_cost(1.9 * INSN_COST);
9950 format %{ "negw $dst, $src1, LSR $src2" %}
9951
9952 ins_encode %{
9953 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9954 Assembler::LSR, $src2$$constant & 0x1f);
9955 %}
9956
9957 ins_pipe(ialu_reg_shift);
9958 %}
9959
9960 // This pattern is automatically generated from aarch64_ad.m4
9961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9962 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9963 immI0 zero, iRegIorL2I src1, immI src2) %{
9964 match(Set dst (SubI zero (RShiftI src1 src2)));
9965
9966 ins_cost(1.9 * INSN_COST);
9967 format %{ "negw $dst, $src1, ASR $src2" %}
9968
9969 ins_encode %{
9970 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9971 Assembler::ASR, $src2$$constant & 0x1f);
9972 %}
9973
9974 ins_pipe(ialu_reg_shift);
9975 %}
9976
9977 // This pattern is automatically generated from aarch64_ad.m4
9978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9979 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9980 immI0 zero, iRegIorL2I src1, immI src2) %{
9981 match(Set dst (SubI zero (LShiftI src1 src2)));
9982
9983 ins_cost(1.9 * INSN_COST);
9984 format %{ "negw $dst, $src1, LSL $src2" %}
9985
9986 ins_encode %{
9987 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9988 Assembler::LSL, $src2$$constant & 0x1f);
9989 %}
9990
9991 ins_pipe(ialu_reg_shift);
9992 %}
9993
9994 // This pattern is automatically generated from aarch64_ad.m4
9995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9996 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9997 immL0 zero, iRegL src1, immI src2) %{
9998 match(Set dst (SubL zero (URShiftL src1 src2)));
9999
10000 ins_cost(1.9 * INSN_COST);
10001 format %{ "neg $dst, $src1, LSR $src2" %}
10002
10003 ins_encode %{
10004 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10005 Assembler::LSR, $src2$$constant & 0x3f);
10006 %}
10007
10008 ins_pipe(ialu_reg_shift);
10009 %}
10010
10011 // This pattern is automatically generated from aarch64_ad.m4
10012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10013 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10014 immL0 zero, iRegL src1, immI src2) %{
10015 match(Set dst (SubL zero (RShiftL src1 src2)));
10016
10017 ins_cost(1.9 * INSN_COST);
10018 format %{ "neg $dst, $src1, ASR $src2" %}
10019
10020 ins_encode %{
10021 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10022 Assembler::ASR, $src2$$constant & 0x3f);
10023 %}
10024
10025 ins_pipe(ialu_reg_shift);
10026 %}
10027
10028 // This pattern is automatically generated from aarch64_ad.m4
10029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10030 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10031 immL0 zero, iRegL src1, immI src2) %{
10032 match(Set dst (SubL zero (LShiftL src1 src2)));
10033
10034 ins_cost(1.9 * INSN_COST);
10035 format %{ "neg $dst, $src1, LSL $src2" %}
10036
10037 ins_encode %{
10038 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10039 Assembler::LSL, $src2$$constant & 0x3f);
10040 %}
10041
10042 ins_pipe(ialu_reg_shift);
10043 %}
10044
10045 // This pattern is automatically generated from aarch64_ad.m4
10046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10047 instruct AndI_reg_not_reg(iRegINoSp dst,
10048 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10049 match(Set dst (AndI src1 (XorI src2 m1)));
10050 ins_cost(INSN_COST);
10051 format %{ "bicw $dst, $src1, $src2" %}
10052
10053 ins_encode %{
10054 __ bicw(as_Register($dst$$reg),
10055 as_Register($src1$$reg),
10056 as_Register($src2$$reg),
10057 Assembler::LSL, 0);
10058 %}
10059
10060 ins_pipe(ialu_reg_reg);
10061 %}
10062
10063 // This pattern is automatically generated from aarch64_ad.m4
10064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10065 instruct AndL_reg_not_reg(iRegLNoSp dst,
10066 iRegL src1, iRegL src2, immL_M1 m1) %{
10067 match(Set dst (AndL src1 (XorL src2 m1)));
10068 ins_cost(INSN_COST);
10069 format %{ "bic $dst, $src1, $src2" %}
10070
10071 ins_encode %{
10072 __ bic(as_Register($dst$$reg),
10073 as_Register($src1$$reg),
10074 as_Register($src2$$reg),
10075 Assembler::LSL, 0);
10076 %}
10077
10078 ins_pipe(ialu_reg_reg);
10079 %}
10080
10081 // This pattern is automatically generated from aarch64_ad.m4
10082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10083 instruct OrI_reg_not_reg(iRegINoSp dst,
10084 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10085 match(Set dst (OrI src1 (XorI src2 m1)));
10086 ins_cost(INSN_COST);
10087 format %{ "ornw $dst, $src1, $src2" %}
10088
10089 ins_encode %{
10090 __ ornw(as_Register($dst$$reg),
10091 as_Register($src1$$reg),
10092 as_Register($src2$$reg),
10093 Assembler::LSL, 0);
10094 %}
10095
10096 ins_pipe(ialu_reg_reg);
10097 %}
10098
10099 // This pattern is automatically generated from aarch64_ad.m4
10100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10101 instruct OrL_reg_not_reg(iRegLNoSp dst,
10102 iRegL src1, iRegL src2, immL_M1 m1) %{
10103 match(Set dst (OrL src1 (XorL src2 m1)));
10104 ins_cost(INSN_COST);
10105 format %{ "orn $dst, $src1, $src2" %}
10106
10107 ins_encode %{
10108 __ orn(as_Register($dst$$reg),
10109 as_Register($src1$$reg),
10110 as_Register($src2$$reg),
10111 Assembler::LSL, 0);
10112 %}
10113
10114 ins_pipe(ialu_reg_reg);
10115 %}
10116
10117 // This pattern is automatically generated from aarch64_ad.m4
10118 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10119 instruct XorI_reg_not_reg(iRegINoSp dst,
10120 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10121 match(Set dst (XorI m1 (XorI src2 src1)));
10122 ins_cost(INSN_COST);
10123 format %{ "eonw $dst, $src1, $src2" %}
10124
10125 ins_encode %{
10126 __ eonw(as_Register($dst$$reg),
10127 as_Register($src1$$reg),
10128 as_Register($src2$$reg),
10129 Assembler::LSL, 0);
10130 %}
10131
10132 ins_pipe(ialu_reg_reg);
10133 %}
10134
10135 // This pattern is automatically generated from aarch64_ad.m4
10136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10137 instruct XorL_reg_not_reg(iRegLNoSp dst,
10138 iRegL src1, iRegL src2, immL_M1 m1) %{
10139 match(Set dst (XorL m1 (XorL src2 src1)));
10140 ins_cost(INSN_COST);
10141 format %{ "eon $dst, $src1, $src2" %}
10142
10143 ins_encode %{
10144 __ eon(as_Register($dst$$reg),
10145 as_Register($src1$$reg),
10146 as_Register($src2$$reg),
10147 Assembler::LSL, 0);
10148 %}
10149
10150 ins_pipe(ialu_reg_reg);
10151 %}
10152
10153 // This pattern is automatically generated from aarch64_ad.m4
10154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10155 // val & (-1 ^ (val >>> shift)) ==> bicw
10156 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10157 iRegIorL2I src1, iRegIorL2I src2,
10158 immI src3, immI_M1 src4) %{
10159 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10160 ins_cost(1.9 * INSN_COST);
10161 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10162
10163 ins_encode %{
10164 __ bicw(as_Register($dst$$reg),
10165 as_Register($src1$$reg),
10166 as_Register($src2$$reg),
10167 Assembler::LSR,
10168 $src3$$constant & 0x1f);
10169 %}
10170
10171 ins_pipe(ialu_reg_reg_shift);
10172 %}
10173
10174 // This pattern is automatically generated from aarch64_ad.m4
10175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10176 // val & (-1 ^ (val >>> shift)) ==> bic
10177 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10178 iRegL src1, iRegL src2,
10179 immI src3, immL_M1 src4) %{
10180 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10181 ins_cost(1.9 * INSN_COST);
10182 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10183
10184 ins_encode %{
10185 __ bic(as_Register($dst$$reg),
10186 as_Register($src1$$reg),
10187 as_Register($src2$$reg),
10188 Assembler::LSR,
10189 $src3$$constant & 0x3f);
10190 %}
10191
10192 ins_pipe(ialu_reg_reg_shift);
10193 %}
10194
10195 // This pattern is automatically generated from aarch64_ad.m4
10196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10197 // val & (-1 ^ (val >> shift)) ==> bicw
10198 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10199 iRegIorL2I src1, iRegIorL2I src2,
10200 immI src3, immI_M1 src4) %{
10201 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10202 ins_cost(1.9 * INSN_COST);
10203 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10204
10205 ins_encode %{
10206 __ bicw(as_Register($dst$$reg),
10207 as_Register($src1$$reg),
10208 as_Register($src2$$reg),
10209 Assembler::ASR,
10210 $src3$$constant & 0x1f);
10211 %}
10212
10213 ins_pipe(ialu_reg_reg_shift);
10214 %}
10215
10216 // This pattern is automatically generated from aarch64_ad.m4
10217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10218 // val & (-1 ^ (val >> shift)) ==> bic
10219 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10220 iRegL src1, iRegL src2,
10221 immI src3, immL_M1 src4) %{
10222 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10223 ins_cost(1.9 * INSN_COST);
10224 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10225
10226 ins_encode %{
10227 __ bic(as_Register($dst$$reg),
10228 as_Register($src1$$reg),
10229 as_Register($src2$$reg),
10230 Assembler::ASR,
10231 $src3$$constant & 0x3f);
10232 %}
10233
10234 ins_pipe(ialu_reg_reg_shift);
10235 %}
10236
10237 // This pattern is automatically generated from aarch64_ad.m4
10238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10239 // val & (-1 ^ (val ror shift)) ==> bicw
10240 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10241 iRegIorL2I src1, iRegIorL2I src2,
10242 immI src3, immI_M1 src4) %{
10243 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10244 ins_cost(1.9 * INSN_COST);
10245 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10246
10247 ins_encode %{
10248 __ bicw(as_Register($dst$$reg),
10249 as_Register($src1$$reg),
10250 as_Register($src2$$reg),
10251 Assembler::ROR,
10252 $src3$$constant & 0x1f);
10253 %}
10254
10255 ins_pipe(ialu_reg_reg_shift);
10256 %}
10257
10258 // This pattern is automatically generated from aarch64_ad.m4
10259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10260 // val & (-1 ^ (val ror shift)) ==> bic
10261 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10262 iRegL src1, iRegL src2,
10263 immI src3, immL_M1 src4) %{
10264 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10265 ins_cost(1.9 * INSN_COST);
10266 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10267
10268 ins_encode %{
10269 __ bic(as_Register($dst$$reg),
10270 as_Register($src1$$reg),
10271 as_Register($src2$$reg),
10272 Assembler::ROR,
10273 $src3$$constant & 0x3f);
10274 %}
10275
10276 ins_pipe(ialu_reg_reg_shift);
10277 %}
10278
10279 // This pattern is automatically generated from aarch64_ad.m4
10280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10281 // val & (-1 ^ (val << shift)) ==> bicw
10282 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10283 iRegIorL2I src1, iRegIorL2I src2,
10284 immI src3, immI_M1 src4) %{
10285 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10286 ins_cost(1.9 * INSN_COST);
10287 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10288
10289 ins_encode %{
10290 __ bicw(as_Register($dst$$reg),
10291 as_Register($src1$$reg),
10292 as_Register($src2$$reg),
10293 Assembler::LSL,
10294 $src3$$constant & 0x1f);
10295 %}
10296
10297 ins_pipe(ialu_reg_reg_shift);
10298 %}
10299
10300 // This pattern is automatically generated from aarch64_ad.m4
10301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10302 // val & (-1 ^ (val << shift)) ==> bic
10303 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10304 iRegL src1, iRegL src2,
10305 immI src3, immL_M1 src4) %{
10306 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10307 ins_cost(1.9 * INSN_COST);
10308 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10309
10310 ins_encode %{
10311 __ bic(as_Register($dst$$reg),
10312 as_Register($src1$$reg),
10313 as_Register($src2$$reg),
10314 Assembler::LSL,
10315 $src3$$constant & 0x3f);
10316 %}
10317
10318 ins_pipe(ialu_reg_reg_shift);
10319 %}
10320
10321 // This pattern is automatically generated from aarch64_ad.m4
10322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10323 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10324 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10325 iRegIorL2I src1, iRegIorL2I src2,
10326 immI src3, immI_M1 src4) %{
10327 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10328 ins_cost(1.9 * INSN_COST);
10329 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10330
10331 ins_encode %{
10332 __ eonw(as_Register($dst$$reg),
10333 as_Register($src1$$reg),
10334 as_Register($src2$$reg),
10335 Assembler::LSR,
10336 $src3$$constant & 0x1f);
10337 %}
10338
10339 ins_pipe(ialu_reg_reg_shift);
10340 %}
10341
10342 // This pattern is automatically generated from aarch64_ad.m4
10343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10344 // val ^ (-1 ^ (val >>> shift)) ==> eon
10345 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10346 iRegL src1, iRegL src2,
10347 immI src3, immL_M1 src4) %{
10348 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10349 ins_cost(1.9 * INSN_COST);
10350 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10351
10352 ins_encode %{
10353 __ eon(as_Register($dst$$reg),
10354 as_Register($src1$$reg),
10355 as_Register($src2$$reg),
10356 Assembler::LSR,
10357 $src3$$constant & 0x3f);
10358 %}
10359
10360 ins_pipe(ialu_reg_reg_shift);
10361 %}
10362
10363 // This pattern is automatically generated from aarch64_ad.m4
10364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10365 // val ^ (-1 ^ (val >> shift)) ==> eonw
10366 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10367 iRegIorL2I src1, iRegIorL2I src2,
10368 immI src3, immI_M1 src4) %{
10369 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10370 ins_cost(1.9 * INSN_COST);
10371 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10372
10373 ins_encode %{
10374 __ eonw(as_Register($dst$$reg),
10375 as_Register($src1$$reg),
10376 as_Register($src2$$reg),
10377 Assembler::ASR,
10378 $src3$$constant & 0x1f);
10379 %}
10380
10381 ins_pipe(ialu_reg_reg_shift);
10382 %}
10383
10384 // This pattern is automatically generated from aarch64_ad.m4
10385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10386 // val ^ (-1 ^ (val >> shift)) ==> eon
10387 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10388 iRegL src1, iRegL src2,
10389 immI src3, immL_M1 src4) %{
10390 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10391 ins_cost(1.9 * INSN_COST);
10392 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10393
10394 ins_encode %{
10395 __ eon(as_Register($dst$$reg),
10396 as_Register($src1$$reg),
10397 as_Register($src2$$reg),
10398 Assembler::ASR,
10399 $src3$$constant & 0x3f);
10400 %}
10401
10402 ins_pipe(ialu_reg_reg_shift);
10403 %}
10404
10405 // This pattern is automatically generated from aarch64_ad.m4
10406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10407 // val ^ (-1 ^ (val ror shift)) ==> eonw
10408 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10409 iRegIorL2I src1, iRegIorL2I src2,
10410 immI src3, immI_M1 src4) %{
10411 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10412 ins_cost(1.9 * INSN_COST);
10413 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10414
10415 ins_encode %{
10416 __ eonw(as_Register($dst$$reg),
10417 as_Register($src1$$reg),
10418 as_Register($src2$$reg),
10419 Assembler::ROR,
10420 $src3$$constant & 0x1f);
10421 %}
10422
10423 ins_pipe(ialu_reg_reg_shift);
10424 %}
10425
10426 // This pattern is automatically generated from aarch64_ad.m4
10427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10428 // val ^ (-1 ^ (val ror shift)) ==> eon
10429 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10430 iRegL src1, iRegL src2,
10431 immI src3, immL_M1 src4) %{
10432 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10433 ins_cost(1.9 * INSN_COST);
10434 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10435
10436 ins_encode %{
10437 __ eon(as_Register($dst$$reg),
10438 as_Register($src1$$reg),
10439 as_Register($src2$$reg),
10440 Assembler::ROR,
10441 $src3$$constant & 0x3f);
10442 %}
10443
10444 ins_pipe(ialu_reg_reg_shift);
10445 %}
10446
10447 // This pattern is automatically generated from aarch64_ad.m4
10448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10449 // val ^ (-1 ^ (val << shift)) ==> eonw
10450 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10451 iRegIorL2I src1, iRegIorL2I src2,
10452 immI src3, immI_M1 src4) %{
10453 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10454 ins_cost(1.9 * INSN_COST);
10455 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10456
10457 ins_encode %{
10458 __ eonw(as_Register($dst$$reg),
10459 as_Register($src1$$reg),
10460 as_Register($src2$$reg),
10461 Assembler::LSL,
10462 $src3$$constant & 0x1f);
10463 %}
10464
10465 ins_pipe(ialu_reg_reg_shift);
10466 %}
10467
10468 // This pattern is automatically generated from aarch64_ad.m4
10469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10470 // val ^ (-1 ^ (val << shift)) ==> eon
10471 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10472 iRegL src1, iRegL src2,
10473 immI src3, immL_M1 src4) %{
10474 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10475 ins_cost(1.9 * INSN_COST);
10476 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10477
10478 ins_encode %{
10479 __ eon(as_Register($dst$$reg),
10480 as_Register($src1$$reg),
10481 as_Register($src2$$reg),
10482 Assembler::LSL,
10483 $src3$$constant & 0x3f);
10484 %}
10485
10486 ins_pipe(ialu_reg_reg_shift);
10487 %}
10488
10489 // This pattern is automatically generated from aarch64_ad.m4
10490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10491 // val | (-1 ^ (val >>> shift)) ==> ornw
10492 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10493 iRegIorL2I src1, iRegIorL2I src2,
10494 immI src3, immI_M1 src4) %{
10495 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10496 ins_cost(1.9 * INSN_COST);
10497 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10498
10499 ins_encode %{
10500 __ ornw(as_Register($dst$$reg),
10501 as_Register($src1$$reg),
10502 as_Register($src2$$reg),
10503 Assembler::LSR,
10504 $src3$$constant & 0x1f);
10505 %}
10506
10507 ins_pipe(ialu_reg_reg_shift);
10508 %}
10509
10510 // This pattern is automatically generated from aarch64_ad.m4
10511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10512 // val | (-1 ^ (val >>> shift)) ==> orn
10513 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10514 iRegL src1, iRegL src2,
10515 immI src3, immL_M1 src4) %{
10516 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10517 ins_cost(1.9 * INSN_COST);
10518 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10519
10520 ins_encode %{
10521 __ orn(as_Register($dst$$reg),
10522 as_Register($src1$$reg),
10523 as_Register($src2$$reg),
10524 Assembler::LSR,
10525 $src3$$constant & 0x3f);
10526 %}
10527
10528 ins_pipe(ialu_reg_reg_shift);
10529 %}
10530
10531 // This pattern is automatically generated from aarch64_ad.m4
10532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10533 // val | (-1 ^ (val >> shift)) ==> ornw
10534 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10535 iRegIorL2I src1, iRegIorL2I src2,
10536 immI src3, immI_M1 src4) %{
10537 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10538 ins_cost(1.9 * INSN_COST);
10539 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10540
10541 ins_encode %{
10542 __ ornw(as_Register($dst$$reg),
10543 as_Register($src1$$reg),
10544 as_Register($src2$$reg),
10545 Assembler::ASR,
10546 $src3$$constant & 0x1f);
10547 %}
10548
10549 ins_pipe(ialu_reg_reg_shift);
10550 %}
10551
10552 // This pattern is automatically generated from aarch64_ad.m4
10553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10554 // val | (-1 ^ (val >> shift)) ==> orn
10555 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10556 iRegL src1, iRegL src2,
10557 immI src3, immL_M1 src4) %{
10558 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10559 ins_cost(1.9 * INSN_COST);
10560 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10561
10562 ins_encode %{
10563 __ orn(as_Register($dst$$reg),
10564 as_Register($src1$$reg),
10565 as_Register($src2$$reg),
10566 Assembler::ASR,
10567 $src3$$constant & 0x3f);
10568 %}
10569
10570 ins_pipe(ialu_reg_reg_shift);
10571 %}
10572
10573 // This pattern is automatically generated from aarch64_ad.m4
10574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10575 // val | (-1 ^ (val ror shift)) ==> ornw
10576 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10577 iRegIorL2I src1, iRegIorL2I src2,
10578 immI src3, immI_M1 src4) %{
10579 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10580 ins_cost(1.9 * INSN_COST);
10581 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10582
10583 ins_encode %{
10584 __ ornw(as_Register($dst$$reg),
10585 as_Register($src1$$reg),
10586 as_Register($src2$$reg),
10587 Assembler::ROR,
10588 $src3$$constant & 0x1f);
10589 %}
10590
10591 ins_pipe(ialu_reg_reg_shift);
10592 %}
10593
10594 // This pattern is automatically generated from aarch64_ad.m4
10595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10596 // val | (-1 ^ (val ror shift)) ==> orn
10597 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10598 iRegL src1, iRegL src2,
10599 immI src3, immL_M1 src4) %{
10600 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10601 ins_cost(1.9 * INSN_COST);
10602 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10603
10604 ins_encode %{
10605 __ orn(as_Register($dst$$reg),
10606 as_Register($src1$$reg),
10607 as_Register($src2$$reg),
10608 Assembler::ROR,
10609 $src3$$constant & 0x3f);
10610 %}
10611
10612 ins_pipe(ialu_reg_reg_shift);
10613 %}
10614
10615 // This pattern is automatically generated from aarch64_ad.m4
10616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10617 // val | (-1 ^ (val << shift)) ==> ornw
10618 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10619 iRegIorL2I src1, iRegIorL2I src2,
10620 immI src3, immI_M1 src4) %{
10621 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10622 ins_cost(1.9 * INSN_COST);
10623 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10624
10625 ins_encode %{
10626 __ ornw(as_Register($dst$$reg),
10627 as_Register($src1$$reg),
10628 as_Register($src2$$reg),
10629 Assembler::LSL,
10630 $src3$$constant & 0x1f);
10631 %}
10632
10633 ins_pipe(ialu_reg_reg_shift);
10634 %}
10635
10636 // This pattern is automatically generated from aarch64_ad.m4
10637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10638 // val | (-1 ^ (val << shift)) ==> orn
10639 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10640 iRegL src1, iRegL src2,
10641 immI src3, immL_M1 src4) %{
10642 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10643 ins_cost(1.9 * INSN_COST);
10644 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10645
10646 ins_encode %{
10647 __ orn(as_Register($dst$$reg),
10648 as_Register($src1$$reg),
10649 as_Register($src2$$reg),
10650 Assembler::LSL,
10651 $src3$$constant & 0x3f);
10652 %}
10653
10654 ins_pipe(ialu_reg_reg_shift);
10655 %}
10656
10657 // This pattern is automatically generated from aarch64_ad.m4
10658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10659 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10660 iRegIorL2I src1, iRegIorL2I src2,
10661 immI src3) %{
10662 match(Set dst (AndI src1 (URShiftI src2 src3)));
10663
10664 ins_cost(1.9 * INSN_COST);
10665 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10666
10667 ins_encode %{
10668 __ andw(as_Register($dst$$reg),
10669 as_Register($src1$$reg),
10670 as_Register($src2$$reg),
10671 Assembler::LSR,
10672 $src3$$constant & 0x1f);
10673 %}
10674
10675 ins_pipe(ialu_reg_reg_shift);
10676 %}
10677
10678 // This pattern is automatically generated from aarch64_ad.m4
10679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10680 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10681 iRegL src1, iRegL src2,
10682 immI src3) %{
10683 match(Set dst (AndL src1 (URShiftL src2 src3)));
10684
10685 ins_cost(1.9 * INSN_COST);
10686 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10687
10688 ins_encode %{
10689 __ andr(as_Register($dst$$reg),
10690 as_Register($src1$$reg),
10691 as_Register($src2$$reg),
10692 Assembler::LSR,
10693 $src3$$constant & 0x3f);
10694 %}
10695
10696 ins_pipe(ialu_reg_reg_shift);
10697 %}
10698
10699 // This pattern is automatically generated from aarch64_ad.m4
10700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10701 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10702 iRegIorL2I src1, iRegIorL2I src2,
10703 immI src3) %{
10704 match(Set dst (AndI src1 (RShiftI src2 src3)));
10705
10706 ins_cost(1.9 * INSN_COST);
10707 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10708
10709 ins_encode %{
10710 __ andw(as_Register($dst$$reg),
10711 as_Register($src1$$reg),
10712 as_Register($src2$$reg),
10713 Assembler::ASR,
10714 $src3$$constant & 0x1f);
10715 %}
10716
10717 ins_pipe(ialu_reg_reg_shift);
10718 %}
10719
10720 // This pattern is automatically generated from aarch64_ad.m4
10721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10722 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10723 iRegL src1, iRegL src2,
10724 immI src3) %{
10725 match(Set dst (AndL src1 (RShiftL src2 src3)));
10726
10727 ins_cost(1.9 * INSN_COST);
10728 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10729
10730 ins_encode %{
10731 __ andr(as_Register($dst$$reg),
10732 as_Register($src1$$reg),
10733 as_Register($src2$$reg),
10734 Assembler::ASR,
10735 $src3$$constant & 0x3f);
10736 %}
10737
10738 ins_pipe(ialu_reg_reg_shift);
10739 %}
10740
10741 // This pattern is automatically generated from aarch64_ad.m4
10742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10743 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10744 iRegIorL2I src1, iRegIorL2I src2,
10745 immI src3) %{
10746 match(Set dst (AndI src1 (LShiftI src2 src3)));
10747
10748 ins_cost(1.9 * INSN_COST);
10749 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10750
10751 ins_encode %{
10752 __ andw(as_Register($dst$$reg),
10753 as_Register($src1$$reg),
10754 as_Register($src2$$reg),
10755 Assembler::LSL,
10756 $src3$$constant & 0x1f);
10757 %}
10758
10759 ins_pipe(ialu_reg_reg_shift);
10760 %}
10761
10762 // This pattern is automatically generated from aarch64_ad.m4
10763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10764 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10765 iRegL src1, iRegL src2,
10766 immI src3) %{
10767 match(Set dst (AndL src1 (LShiftL src2 src3)));
10768
10769 ins_cost(1.9 * INSN_COST);
10770 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10771
10772 ins_encode %{
10773 __ andr(as_Register($dst$$reg),
10774 as_Register($src1$$reg),
10775 as_Register($src2$$reg),
10776 Assembler::LSL,
10777 $src3$$constant & 0x3f);
10778 %}
10779
10780 ins_pipe(ialu_reg_reg_shift);
10781 %}
10782
10783 // This pattern is automatically generated from aarch64_ad.m4
10784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10785 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10786 iRegIorL2I src1, iRegIorL2I src2,
10787 immI src3) %{
10788 match(Set dst (AndI src1 (RotateRight src2 src3)));
10789
10790 ins_cost(1.9 * INSN_COST);
10791 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10792
10793 ins_encode %{
10794 __ andw(as_Register($dst$$reg),
10795 as_Register($src1$$reg),
10796 as_Register($src2$$reg),
10797 Assembler::ROR,
10798 $src3$$constant & 0x1f);
10799 %}
10800
10801 ins_pipe(ialu_reg_reg_shift);
10802 %}
10803
10804 // This pattern is automatically generated from aarch64_ad.m4
10805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10806 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10807 iRegL src1, iRegL src2,
10808 immI src3) %{
10809 match(Set dst (AndL src1 (RotateRight src2 src3)));
10810
10811 ins_cost(1.9 * INSN_COST);
10812 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10813
10814 ins_encode %{
10815 __ andr(as_Register($dst$$reg),
10816 as_Register($src1$$reg),
10817 as_Register($src2$$reg),
10818 Assembler::ROR,
10819 $src3$$constant & 0x3f);
10820 %}
10821
10822 ins_pipe(ialu_reg_reg_shift);
10823 %}
10824
10825 // This pattern is automatically generated from aarch64_ad.m4
10826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10827 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10828 iRegIorL2I src1, iRegIorL2I src2,
10829 immI src3) %{
10830 match(Set dst (XorI src1 (URShiftI src2 src3)));
10831
10832 ins_cost(1.9 * INSN_COST);
10833 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10834
10835 ins_encode %{
10836 __ eorw(as_Register($dst$$reg),
10837 as_Register($src1$$reg),
10838 as_Register($src2$$reg),
10839 Assembler::LSR,
10840 $src3$$constant & 0x1f);
10841 %}
10842
10843 ins_pipe(ialu_reg_reg_shift);
10844 %}
10845
10846 // This pattern is automatically generated from aarch64_ad.m4
10847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10848 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10849 iRegL src1, iRegL src2,
10850 immI src3) %{
10851 match(Set dst (XorL src1 (URShiftL src2 src3)));
10852
10853 ins_cost(1.9 * INSN_COST);
10854 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10855
10856 ins_encode %{
10857 __ eor(as_Register($dst$$reg),
10858 as_Register($src1$$reg),
10859 as_Register($src2$$reg),
10860 Assembler::LSR,
10861 $src3$$constant & 0x3f);
10862 %}
10863
10864 ins_pipe(ialu_reg_reg_shift);
10865 %}
10866
10867 // This pattern is automatically generated from aarch64_ad.m4
10868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10869 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10870 iRegIorL2I src1, iRegIorL2I src2,
10871 immI src3) %{
10872 match(Set dst (XorI src1 (RShiftI src2 src3)));
10873
10874 ins_cost(1.9 * INSN_COST);
10875 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10876
10877 ins_encode %{
10878 __ eorw(as_Register($dst$$reg),
10879 as_Register($src1$$reg),
10880 as_Register($src2$$reg),
10881 Assembler::ASR,
10882 $src3$$constant & 0x1f);
10883 %}
10884
10885 ins_pipe(ialu_reg_reg_shift);
10886 %}
10887
10888 // This pattern is automatically generated from aarch64_ad.m4
10889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10890 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10891 iRegL src1, iRegL src2,
10892 immI src3) %{
10893 match(Set dst (XorL src1 (RShiftL src2 src3)));
10894
10895 ins_cost(1.9 * INSN_COST);
10896 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10897
10898 ins_encode %{
10899 __ eor(as_Register($dst$$reg),
10900 as_Register($src1$$reg),
10901 as_Register($src2$$reg),
10902 Assembler::ASR,
10903 $src3$$constant & 0x3f);
10904 %}
10905
10906 ins_pipe(ialu_reg_reg_shift);
10907 %}
10908
10909 // This pattern is automatically generated from aarch64_ad.m4
10910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10911 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10912 iRegIorL2I src1, iRegIorL2I src2,
10913 immI src3) %{
10914 match(Set dst (XorI src1 (LShiftI src2 src3)));
10915
10916 ins_cost(1.9 * INSN_COST);
10917 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10918
10919 ins_encode %{
10920 __ eorw(as_Register($dst$$reg),
10921 as_Register($src1$$reg),
10922 as_Register($src2$$reg),
10923 Assembler::LSL,
10924 $src3$$constant & 0x1f);
10925 %}
10926
10927 ins_pipe(ialu_reg_reg_shift);
10928 %}
10929
10930 // This pattern is automatically generated from aarch64_ad.m4
10931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10932 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10933 iRegL src1, iRegL src2,
10934 immI src3) %{
10935 match(Set dst (XorL src1 (LShiftL src2 src3)));
10936
10937 ins_cost(1.9 * INSN_COST);
10938 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10939
10940 ins_encode %{
10941 __ eor(as_Register($dst$$reg),
10942 as_Register($src1$$reg),
10943 as_Register($src2$$reg),
10944 Assembler::LSL,
10945 $src3$$constant & 0x3f);
10946 %}
10947
10948 ins_pipe(ialu_reg_reg_shift);
10949 %}
10950
10951 // This pattern is automatically generated from aarch64_ad.m4
10952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10953 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10954 iRegIorL2I src1, iRegIorL2I src2,
10955 immI src3) %{
10956 match(Set dst (XorI src1 (RotateRight src2 src3)));
10957
10958 ins_cost(1.9 * INSN_COST);
10959 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10960
10961 ins_encode %{
10962 __ eorw(as_Register($dst$$reg),
10963 as_Register($src1$$reg),
10964 as_Register($src2$$reg),
10965 Assembler::ROR,
10966 $src3$$constant & 0x1f);
10967 %}
10968
10969 ins_pipe(ialu_reg_reg_shift);
10970 %}
10971
10972 // This pattern is automatically generated from aarch64_ad.m4
10973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10974 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10975 iRegL src1, iRegL src2,
10976 immI src3) %{
10977 match(Set dst (XorL src1 (RotateRight src2 src3)));
10978
10979 ins_cost(1.9 * INSN_COST);
10980 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10981
10982 ins_encode %{
10983 __ eor(as_Register($dst$$reg),
10984 as_Register($src1$$reg),
10985 as_Register($src2$$reg),
10986 Assembler::ROR,
10987 $src3$$constant & 0x3f);
10988 %}
10989
10990 ins_pipe(ialu_reg_reg_shift);
10991 %}
10992
10993 // This pattern is automatically generated from aarch64_ad.m4
10994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10995 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10996 iRegIorL2I src1, iRegIorL2I src2,
10997 immI src3) %{
10998 match(Set dst (OrI src1 (URShiftI src2 src3)));
10999
11000 ins_cost(1.9 * INSN_COST);
11001 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11002
11003 ins_encode %{
11004 __ orrw(as_Register($dst$$reg),
11005 as_Register($src1$$reg),
11006 as_Register($src2$$reg),
11007 Assembler::LSR,
11008 $src3$$constant & 0x1f);
11009 %}
11010
11011 ins_pipe(ialu_reg_reg_shift);
11012 %}
11013
11014 // This pattern is automatically generated from aarch64_ad.m4
11015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11016 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11017 iRegL src1, iRegL src2,
11018 immI src3) %{
11019 match(Set dst (OrL src1 (URShiftL src2 src3)));
11020
11021 ins_cost(1.9 * INSN_COST);
11022 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11023
11024 ins_encode %{
11025 __ orr(as_Register($dst$$reg),
11026 as_Register($src1$$reg),
11027 as_Register($src2$$reg),
11028 Assembler::LSR,
11029 $src3$$constant & 0x3f);
11030 %}
11031
11032 ins_pipe(ialu_reg_reg_shift);
11033 %}
11034
11035 // This pattern is automatically generated from aarch64_ad.m4
11036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11037 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11038 iRegIorL2I src1, iRegIorL2I src2,
11039 immI src3) %{
11040 match(Set dst (OrI src1 (RShiftI src2 src3)));
11041
11042 ins_cost(1.9 * INSN_COST);
11043 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11044
11045 ins_encode %{
11046 __ orrw(as_Register($dst$$reg),
11047 as_Register($src1$$reg),
11048 as_Register($src2$$reg),
11049 Assembler::ASR,
11050 $src3$$constant & 0x1f);
11051 %}
11052
11053 ins_pipe(ialu_reg_reg_shift);
11054 %}
11055
11056 // This pattern is automatically generated from aarch64_ad.m4
11057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11058 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11059 iRegL src1, iRegL src2,
11060 immI src3) %{
11061 match(Set dst (OrL src1 (RShiftL src2 src3)));
11062
11063 ins_cost(1.9 * INSN_COST);
11064 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11065
11066 ins_encode %{
11067 __ orr(as_Register($dst$$reg),
11068 as_Register($src1$$reg),
11069 as_Register($src2$$reg),
11070 Assembler::ASR,
11071 $src3$$constant & 0x3f);
11072 %}
11073
11074 ins_pipe(ialu_reg_reg_shift);
11075 %}
11076
11077 // This pattern is automatically generated from aarch64_ad.m4
11078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11079 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11080 iRegIorL2I src1, iRegIorL2I src2,
11081 immI src3) %{
11082 match(Set dst (OrI src1 (LShiftI src2 src3)));
11083
11084 ins_cost(1.9 * INSN_COST);
11085 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11086
11087 ins_encode %{
11088 __ orrw(as_Register($dst$$reg),
11089 as_Register($src1$$reg),
11090 as_Register($src2$$reg),
11091 Assembler::LSL,
11092 $src3$$constant & 0x1f);
11093 %}
11094
11095 ins_pipe(ialu_reg_reg_shift);
11096 %}
11097
11098 // This pattern is automatically generated from aarch64_ad.m4
11099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11100 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11101 iRegL src1, iRegL src2,
11102 immI src3) %{
11103 match(Set dst (OrL src1 (LShiftL src2 src3)));
11104
11105 ins_cost(1.9 * INSN_COST);
11106 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11107
11108 ins_encode %{
11109 __ orr(as_Register($dst$$reg),
11110 as_Register($src1$$reg),
11111 as_Register($src2$$reg),
11112 Assembler::LSL,
11113 $src3$$constant & 0x3f);
11114 %}
11115
11116 ins_pipe(ialu_reg_reg_shift);
11117 %}
11118
11119 // This pattern is automatically generated from aarch64_ad.m4
11120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11121 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11122 iRegIorL2I src1, iRegIorL2I src2,
11123 immI src3) %{
11124 match(Set dst (OrI src1 (RotateRight src2 src3)));
11125
11126 ins_cost(1.9 * INSN_COST);
11127 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11128
11129 ins_encode %{
11130 __ orrw(as_Register($dst$$reg),
11131 as_Register($src1$$reg),
11132 as_Register($src2$$reg),
11133 Assembler::ROR,
11134 $src3$$constant & 0x1f);
11135 %}
11136
11137 ins_pipe(ialu_reg_reg_shift);
11138 %}
11139
11140 // This pattern is automatically generated from aarch64_ad.m4
11141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11142 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11143 iRegL src1, iRegL src2,
11144 immI src3) %{
11145 match(Set dst (OrL src1 (RotateRight src2 src3)));
11146
11147 ins_cost(1.9 * INSN_COST);
11148 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11149
11150 ins_encode %{
11151 __ orr(as_Register($dst$$reg),
11152 as_Register($src1$$reg),
11153 as_Register($src2$$reg),
11154 Assembler::ROR,
11155 $src3$$constant & 0x3f);
11156 %}
11157
11158 ins_pipe(ialu_reg_reg_shift);
11159 %}
11160
11161 // This pattern is automatically generated from aarch64_ad.m4
11162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11163 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11164 iRegIorL2I src1, iRegIorL2I src2,
11165 immI src3) %{
11166 match(Set dst (AddI src1 (URShiftI src2 src3)));
11167
11168 ins_cost(1.9 * INSN_COST);
11169 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11170
11171 ins_encode %{
11172 __ addw(as_Register($dst$$reg),
11173 as_Register($src1$$reg),
11174 as_Register($src2$$reg),
11175 Assembler::LSR,
11176 $src3$$constant & 0x1f);
11177 %}
11178
11179 ins_pipe(ialu_reg_reg_shift);
11180 %}
11181
11182 // This pattern is automatically generated from aarch64_ad.m4
11183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11184 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11185 iRegL src1, iRegL src2,
11186 immI src3) %{
11187 match(Set dst (AddL src1 (URShiftL src2 src3)));
11188
11189 ins_cost(1.9 * INSN_COST);
11190 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11191
11192 ins_encode %{
11193 __ add(as_Register($dst$$reg),
11194 as_Register($src1$$reg),
11195 as_Register($src2$$reg),
11196 Assembler::LSR,
11197 $src3$$constant & 0x3f);
11198 %}
11199
11200 ins_pipe(ialu_reg_reg_shift);
11201 %}
11202
11203 // This pattern is automatically generated from aarch64_ad.m4
11204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11205 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11206 iRegIorL2I src1, iRegIorL2I src2,
11207 immI src3) %{
11208 match(Set dst (AddI src1 (RShiftI src2 src3)));
11209
11210 ins_cost(1.9 * INSN_COST);
11211 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11212
11213 ins_encode %{
11214 __ addw(as_Register($dst$$reg),
11215 as_Register($src1$$reg),
11216 as_Register($src2$$reg),
11217 Assembler::ASR,
11218 $src3$$constant & 0x1f);
11219 %}
11220
11221 ins_pipe(ialu_reg_reg_shift);
11222 %}
11223
11224 // This pattern is automatically generated from aarch64_ad.m4
11225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11226 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11227 iRegL src1, iRegL src2,
11228 immI src3) %{
11229 match(Set dst (AddL src1 (RShiftL src2 src3)));
11230
11231 ins_cost(1.9 * INSN_COST);
11232 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11233
11234 ins_encode %{
11235 __ add(as_Register($dst$$reg),
11236 as_Register($src1$$reg),
11237 as_Register($src2$$reg),
11238 Assembler::ASR,
11239 $src3$$constant & 0x3f);
11240 %}
11241
11242 ins_pipe(ialu_reg_reg_shift);
11243 %}
11244
11245 // This pattern is automatically generated from aarch64_ad.m4
11246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11247 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11248 iRegIorL2I src1, iRegIorL2I src2,
11249 immI src3) %{
11250 match(Set dst (AddI src1 (LShiftI src2 src3)));
11251
11252 ins_cost(1.9 * INSN_COST);
11253 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11254
11255 ins_encode %{
11256 __ addw(as_Register($dst$$reg),
11257 as_Register($src1$$reg),
11258 as_Register($src2$$reg),
11259 Assembler::LSL,
11260 $src3$$constant & 0x1f);
11261 %}
11262
11263 ins_pipe(ialu_reg_reg_shift);
11264 %}
11265
11266 // This pattern is automatically generated from aarch64_ad.m4
11267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11268 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11269 iRegL src1, iRegL src2,
11270 immI src3) %{
11271 match(Set dst (AddL src1 (LShiftL src2 src3)));
11272
11273 ins_cost(1.9 * INSN_COST);
11274 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11275
11276 ins_encode %{
11277 __ add(as_Register($dst$$reg),
11278 as_Register($src1$$reg),
11279 as_Register($src2$$reg),
11280 Assembler::LSL,
11281 $src3$$constant & 0x3f);
11282 %}
11283
11284 ins_pipe(ialu_reg_reg_shift);
11285 %}
11286
11287 // This pattern is automatically generated from aarch64_ad.m4
11288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11289 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11290 iRegIorL2I src1, iRegIorL2I src2,
11291 immI src3) %{
11292 match(Set dst (SubI src1 (URShiftI src2 src3)));
11293
11294 ins_cost(1.9 * INSN_COST);
11295 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11296
11297 ins_encode %{
11298 __ subw(as_Register($dst$$reg),
11299 as_Register($src1$$reg),
11300 as_Register($src2$$reg),
11301 Assembler::LSR,
11302 $src3$$constant & 0x1f);
11303 %}
11304
11305 ins_pipe(ialu_reg_reg_shift);
11306 %}
11307
11308 // This pattern is automatically generated from aarch64_ad.m4
11309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11310 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11311 iRegL src1, iRegL src2,
11312 immI src3) %{
11313 match(Set dst (SubL src1 (URShiftL src2 src3)));
11314
11315 ins_cost(1.9 * INSN_COST);
11316 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11317
11318 ins_encode %{
11319 __ sub(as_Register($dst$$reg),
11320 as_Register($src1$$reg),
11321 as_Register($src2$$reg),
11322 Assembler::LSR,
11323 $src3$$constant & 0x3f);
11324 %}
11325
11326 ins_pipe(ialu_reg_reg_shift);
11327 %}
11328
11329 // This pattern is automatically generated from aarch64_ad.m4
11330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11331 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11332 iRegIorL2I src1, iRegIorL2I src2,
11333 immI src3) %{
11334 match(Set dst (SubI src1 (RShiftI src2 src3)));
11335
11336 ins_cost(1.9 * INSN_COST);
11337 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11338
11339 ins_encode %{
11340 __ subw(as_Register($dst$$reg),
11341 as_Register($src1$$reg),
11342 as_Register($src2$$reg),
11343 Assembler::ASR,
11344 $src3$$constant & 0x1f);
11345 %}
11346
11347 ins_pipe(ialu_reg_reg_shift);
11348 %}
11349
11350 // This pattern is automatically generated from aarch64_ad.m4
11351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11352 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11353 iRegL src1, iRegL src2,
11354 immI src3) %{
11355 match(Set dst (SubL src1 (RShiftL src2 src3)));
11356
11357 ins_cost(1.9 * INSN_COST);
11358 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11359
11360 ins_encode %{
11361 __ sub(as_Register($dst$$reg),
11362 as_Register($src1$$reg),
11363 as_Register($src2$$reg),
11364 Assembler::ASR,
11365 $src3$$constant & 0x3f);
11366 %}
11367
11368 ins_pipe(ialu_reg_reg_shift);
11369 %}
11370
11371 // This pattern is automatically generated from aarch64_ad.m4
11372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11373 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11374 iRegIorL2I src1, iRegIorL2I src2,
11375 immI src3) %{
11376 match(Set dst (SubI src1 (LShiftI src2 src3)));
11377
11378 ins_cost(1.9 * INSN_COST);
11379 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11380
11381 ins_encode %{
11382 __ subw(as_Register($dst$$reg),
11383 as_Register($src1$$reg),
11384 as_Register($src2$$reg),
11385 Assembler::LSL,
11386 $src3$$constant & 0x1f);
11387 %}
11388
11389 ins_pipe(ialu_reg_reg_shift);
11390 %}
11391
11392 // This pattern is automatically generated from aarch64_ad.m4
11393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11394 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11395 iRegL src1, iRegL src2,
11396 immI src3) %{
11397 match(Set dst (SubL src1 (LShiftL src2 src3)));
11398
11399 ins_cost(1.9 * INSN_COST);
11400 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11401
11402 ins_encode %{
11403 __ sub(as_Register($dst$$reg),
11404 as_Register($src1$$reg),
11405 as_Register($src2$$reg),
11406 Assembler::LSL,
11407 $src3$$constant & 0x3f);
11408 %}
11409
11410 ins_pipe(ialu_reg_reg_shift);
11411 %}
11412
11413 // This pattern is automatically generated from aarch64_ad.m4
11414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11415
11416 // Shift Left followed by Shift Right.
11417 // This idiom is used by the compiler for the i2b bytecode etc.
11418 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11419 %{
11420 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11421 ins_cost(INSN_COST * 2);
11422 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11423 ins_encode %{
11424 int lshift = $lshift_count$$constant & 63;
11425 int rshift = $rshift_count$$constant & 63;
11426 int s = 63 - lshift;
11427 int r = (rshift - lshift) & 63;
11428 __ sbfm(as_Register($dst$$reg),
11429 as_Register($src$$reg),
11430 r, s);
11431 %}
11432
11433 ins_pipe(ialu_reg_shift);
11434 %}
11435
11436 // This pattern is automatically generated from aarch64_ad.m4
11437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11438
11439 // Shift Left followed by Shift Right.
11440 // This idiom is used by the compiler for the i2b bytecode etc.
11441 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11442 %{
11443 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11444 ins_cost(INSN_COST * 2);
11445 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11446 ins_encode %{
11447 int lshift = $lshift_count$$constant & 31;
11448 int rshift = $rshift_count$$constant & 31;
11449 int s = 31 - lshift;
11450 int r = (rshift - lshift) & 31;
11451 __ sbfmw(as_Register($dst$$reg),
11452 as_Register($src$$reg),
11453 r, s);
11454 %}
11455
11456 ins_pipe(ialu_reg_shift);
11457 %}
11458
11459 // This pattern is automatically generated from aarch64_ad.m4
11460 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11461
11462 // Shift Left followed by Shift Right.
11463 // This idiom is used by the compiler for the i2b bytecode etc.
11464 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11465 %{
11466 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11467 ins_cost(INSN_COST * 2);
11468 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11469 ins_encode %{
11470 int lshift = $lshift_count$$constant & 63;
11471 int rshift = $rshift_count$$constant & 63;
11472 int s = 63 - lshift;
11473 int r = (rshift - lshift) & 63;
11474 __ ubfm(as_Register($dst$$reg),
11475 as_Register($src$$reg),
11476 r, s);
11477 %}
11478
11479 ins_pipe(ialu_reg_shift);
11480 %}
11481
11482 // This pattern is automatically generated from aarch64_ad.m4
11483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11484
11485 // Shift Left followed by Shift Right.
11486 // This idiom is used by the compiler for the i2b bytecode etc.
11487 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11488 %{
11489 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11490 ins_cost(INSN_COST * 2);
11491 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11492 ins_encode %{
11493 int lshift = $lshift_count$$constant & 31;
11494 int rshift = $rshift_count$$constant & 31;
11495 int s = 31 - lshift;
11496 int r = (rshift - lshift) & 31;
11497 __ ubfmw(as_Register($dst$$reg),
11498 as_Register($src$$reg),
11499 r, s);
11500 %}
11501
11502 ins_pipe(ialu_reg_shift);
11503 %}
11504
11505 // Bitfield extract with shift & mask
11506
11507 // This pattern is automatically generated from aarch64_ad.m4
11508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11509 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11510 %{
11511 match(Set dst (AndI (URShiftI src rshift) mask));
11512 // Make sure we are not going to exceed what ubfxw can do.
11513 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11514
11515 ins_cost(INSN_COST);
11516 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11517 ins_encode %{
11518 int rshift = $rshift$$constant & 31;
11519 intptr_t mask = $mask$$constant;
11520 int width = exact_log2(mask+1);
11521 __ ubfxw(as_Register($dst$$reg),
11522 as_Register($src$$reg), rshift, width);
11523 %}
11524 ins_pipe(ialu_reg_shift);
11525 %}
11526
11527 // This pattern is automatically generated from aarch64_ad.m4
11528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11529 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11530 %{
11531 match(Set dst (AndL (URShiftL src rshift) mask));
11532 // Make sure we are not going to exceed what ubfx can do.
11533 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11534
11535 ins_cost(INSN_COST);
11536 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11537 ins_encode %{
11538 int rshift = $rshift$$constant & 63;
11539 intptr_t mask = $mask$$constant;
11540 int width = exact_log2_long(mask+1);
11541 __ ubfx(as_Register($dst$$reg),
11542 as_Register($src$$reg), rshift, width);
11543 %}
11544 ins_pipe(ialu_reg_shift);
11545 %}
11546
11547
11548 // This pattern is automatically generated from aarch64_ad.m4
11549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11550
11551 // We can use ubfx when extending an And with a mask when we know mask
11552 // is positive. We know that because immI_bitmask guarantees it.
11553 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11554 %{
11555 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11556 // Make sure we are not going to exceed what ubfxw can do.
11557 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11558
11559 ins_cost(INSN_COST * 2);
11560 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11561 ins_encode %{
11562 int rshift = $rshift$$constant & 31;
11563 intptr_t mask = $mask$$constant;
11564 int width = exact_log2(mask+1);
11565 __ ubfx(as_Register($dst$$reg),
11566 as_Register($src$$reg), rshift, width);
11567 %}
11568 ins_pipe(ialu_reg_shift);
11569 %}
11570
11571
11572 // This pattern is automatically generated from aarch64_ad.m4
11573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11574
11575 // We can use ubfiz when masking by a positive number and then left shifting the result.
11576 // We know that the mask is positive because immI_bitmask guarantees it.
11577 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11578 %{
11579 match(Set dst (LShiftI (AndI src mask) lshift));
11580 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11581
11582 ins_cost(INSN_COST);
11583 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11584 ins_encode %{
11585 int lshift = $lshift$$constant & 31;
11586 intptr_t mask = $mask$$constant;
11587 int width = exact_log2(mask+1);
11588 __ ubfizw(as_Register($dst$$reg),
11589 as_Register($src$$reg), lshift, width);
11590 %}
11591 ins_pipe(ialu_reg_shift);
11592 %}
11593
11594 // This pattern is automatically generated from aarch64_ad.m4
11595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11596
11597 // We can use ubfiz when masking by a positive number and then left shifting the result.
11598 // We know that the mask is positive because immL_bitmask guarantees it.
11599 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11600 %{
11601 match(Set dst (LShiftL (AndL src mask) lshift));
11602 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11603
11604 ins_cost(INSN_COST);
11605 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11606 ins_encode %{
11607 int lshift = $lshift$$constant & 63;
11608 intptr_t mask = $mask$$constant;
11609 int width = exact_log2_long(mask+1);
11610 __ ubfiz(as_Register($dst$$reg),
11611 as_Register($src$$reg), lshift, width);
11612 %}
11613 ins_pipe(ialu_reg_shift);
11614 %}
11615
11616 // This pattern is automatically generated from aarch64_ad.m4
11617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11618
11619 // We can use ubfiz when masking by a positive number and then left shifting the result.
11620 // We know that the mask is positive because immI_bitmask guarantees it.
11621 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11622 %{
11623 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11624 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11625
11626 ins_cost(INSN_COST);
11627 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11628 ins_encode %{
11629 int lshift = $lshift$$constant & 31;
11630 intptr_t mask = $mask$$constant;
11631 int width = exact_log2(mask+1);
11632 __ ubfizw(as_Register($dst$$reg),
11633 as_Register($src$$reg), lshift, width);
11634 %}
11635 ins_pipe(ialu_reg_shift);
11636 %}
11637
11638 // This pattern is automatically generated from aarch64_ad.m4
11639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11640
11641 // We can use ubfiz when masking by a positive number and then left shifting the result.
11642 // We know that the mask is positive because immL_bitmask guarantees it.
11643 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11644 %{
11645 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11646 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11647
11648 ins_cost(INSN_COST);
11649 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11650 ins_encode %{
11651 int lshift = $lshift$$constant & 63;
11652 intptr_t mask = $mask$$constant;
11653 int width = exact_log2_long(mask+1);
11654 __ ubfiz(as_Register($dst$$reg),
11655 as_Register($src$$reg), lshift, width);
11656 %}
11657 ins_pipe(ialu_reg_shift);
11658 %}
11659
11660
11661 // This pattern is automatically generated from aarch64_ad.m4
11662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11663
11664 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11665 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11666 %{
11667 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11668 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11669
11670 ins_cost(INSN_COST);
11671 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11672 ins_encode %{
11673 int lshift = $lshift$$constant & 63;
11674 intptr_t mask = $mask$$constant;
11675 int width = exact_log2(mask+1);
11676 __ ubfiz(as_Register($dst$$reg),
11677 as_Register($src$$reg), lshift, width);
11678 %}
11679 ins_pipe(ialu_reg_shift);
11680 %}
11681
11682 // This pattern is automatically generated from aarch64_ad.m4
11683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11684
11685 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11686 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11687 %{
11688 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11689 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11690
11691 ins_cost(INSN_COST);
11692 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11693 ins_encode %{
11694 int lshift = $lshift$$constant & 31;
11695 intptr_t mask = $mask$$constant;
11696 int width = exact_log2(mask+1);
11697 __ ubfiz(as_Register($dst$$reg),
11698 as_Register($src$$reg), lshift, width);
11699 %}
11700 ins_pipe(ialu_reg_shift);
11701 %}
11702
11703 // This pattern is automatically generated from aarch64_ad.m4
11704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11705
11706 // Can skip int2long conversions after AND with small bitmask
11707 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11708 %{
11709 match(Set dst (ConvI2L (AndI src msk)));
11710 ins_cost(INSN_COST);
11711 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11712 ins_encode %{
11713 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11714 %}
11715 ins_pipe(ialu_reg_shift);
11716 %}
11717
11718
11719 // Rotations
11720
11721 // This pattern is automatically generated from aarch64_ad.m4
11722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11723 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11724 %{
11725 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11726 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11727
11728 ins_cost(INSN_COST);
11729 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11730
11731 ins_encode %{
11732 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11733 $rshift$$constant & 63);
11734 %}
11735 ins_pipe(ialu_reg_reg_extr);
11736 %}
11737
11738
11739 // This pattern is automatically generated from aarch64_ad.m4
11740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11741 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11742 %{
11743 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11744 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11745
11746 ins_cost(INSN_COST);
11747 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11748
11749 ins_encode %{
11750 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11751 $rshift$$constant & 31);
11752 %}
11753 ins_pipe(ialu_reg_reg_extr);
11754 %}
11755
11756
11757 // This pattern is automatically generated from aarch64_ad.m4
11758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11759 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11760 %{
11761 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11762 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11763
11764 ins_cost(INSN_COST);
11765 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11766
11767 ins_encode %{
11768 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11769 $rshift$$constant & 63);
11770 %}
11771 ins_pipe(ialu_reg_reg_extr);
11772 %}
11773
11774
11775 // This pattern is automatically generated from aarch64_ad.m4
11776 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11777 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11778 %{
11779 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11780 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11781
11782 ins_cost(INSN_COST);
11783 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11784
11785 ins_encode %{
11786 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11787 $rshift$$constant & 31);
11788 %}
11789 ins_pipe(ialu_reg_reg_extr);
11790 %}
11791
11792 // This pattern is automatically generated from aarch64_ad.m4
11793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11794 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11795 %{
11796 match(Set dst (RotateRight src shift));
11797
11798 ins_cost(INSN_COST);
11799 format %{ "ror $dst, $src, $shift" %}
11800
11801 ins_encode %{
11802 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11803 $shift$$constant & 0x1f);
11804 %}
11805 ins_pipe(ialu_reg_reg_vshift);
11806 %}
11807
11808 // This pattern is automatically generated from aarch64_ad.m4
11809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11810 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11811 %{
11812 match(Set dst (RotateRight src shift));
11813
11814 ins_cost(INSN_COST);
11815 format %{ "ror $dst, $src, $shift" %}
11816
11817 ins_encode %{
11818 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11819 $shift$$constant & 0x3f);
11820 %}
11821 ins_pipe(ialu_reg_reg_vshift);
11822 %}
11823
11824 // This pattern is automatically generated from aarch64_ad.m4
11825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11826 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11827 %{
11828 match(Set dst (RotateRight src shift));
11829
11830 ins_cost(INSN_COST);
11831 format %{ "ror $dst, $src, $shift" %}
11832
11833 ins_encode %{
11834 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11835 %}
11836 ins_pipe(ialu_reg_reg_vshift);
11837 %}
11838
11839 // This pattern is automatically generated from aarch64_ad.m4
11840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11841 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11842 %{
11843 match(Set dst (RotateRight src shift));
11844
11845 ins_cost(INSN_COST);
11846 format %{ "ror $dst, $src, $shift" %}
11847
11848 ins_encode %{
11849 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11850 %}
11851 ins_pipe(ialu_reg_reg_vshift);
11852 %}
11853
11854 // This pattern is automatically generated from aarch64_ad.m4
11855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11856 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11857 %{
11858 match(Set dst (RotateLeft src shift));
11859
11860 ins_cost(INSN_COST);
11861 format %{ "rol $dst, $src, $shift" %}
11862
11863 ins_encode %{
11864 __ subw(rscratch1, zr, as_Register($shift$$reg));
11865 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11866 %}
11867 ins_pipe(ialu_reg_reg_vshift);
11868 %}
11869
11870 // This pattern is automatically generated from aarch64_ad.m4
11871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11872 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11873 %{
11874 match(Set dst (RotateLeft src shift));
11875
11876 ins_cost(INSN_COST);
11877 format %{ "rol $dst, $src, $shift" %}
11878
11879 ins_encode %{
11880 __ subw(rscratch1, zr, as_Register($shift$$reg));
11881 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11882 %}
11883 ins_pipe(ialu_reg_reg_vshift);
11884 %}
11885
11886
11887 // Add/subtract (extended)
11888
11889 // This pattern is automatically generated from aarch64_ad.m4
11890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11891 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11892 %{
11893 match(Set dst (AddL src1 (ConvI2L src2)));
11894 ins_cost(INSN_COST);
11895 format %{ "add $dst, $src1, $src2, sxtw" %}
11896
11897 ins_encode %{
11898 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11899 as_Register($src2$$reg), ext::sxtw);
11900 %}
11901 ins_pipe(ialu_reg_reg);
11902 %}
11903
11904 // This pattern is automatically generated from aarch64_ad.m4
11905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11906 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11907 %{
11908 match(Set dst (SubL src1 (ConvI2L src2)));
11909 ins_cost(INSN_COST);
11910 format %{ "sub $dst, $src1, $src2, sxtw" %}
11911
11912 ins_encode %{
11913 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11914 as_Register($src2$$reg), ext::sxtw);
11915 %}
11916 ins_pipe(ialu_reg_reg);
11917 %}
11918
11919 // This pattern is automatically generated from aarch64_ad.m4
11920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11921 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11922 %{
11923 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11924 ins_cost(INSN_COST);
11925 format %{ "add $dst, $src1, $src2, sxth" %}
11926
11927 ins_encode %{
11928 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11929 as_Register($src2$$reg), ext::sxth);
11930 %}
11931 ins_pipe(ialu_reg_reg);
11932 %}
11933
11934 // This pattern is automatically generated from aarch64_ad.m4
11935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11936 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11937 %{
11938 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11939 ins_cost(INSN_COST);
11940 format %{ "add $dst, $src1, $src2, sxtb" %}
11941
11942 ins_encode %{
11943 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11944 as_Register($src2$$reg), ext::sxtb);
11945 %}
11946 ins_pipe(ialu_reg_reg);
11947 %}
11948
11949 // This pattern is automatically generated from aarch64_ad.m4
11950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11951 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11952 %{
11953 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11954 ins_cost(INSN_COST);
11955 format %{ "add $dst, $src1, $src2, uxtb" %}
11956
11957 ins_encode %{
11958 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11959 as_Register($src2$$reg), ext::uxtb);
11960 %}
11961 ins_pipe(ialu_reg_reg);
11962 %}
11963
11964 // This pattern is automatically generated from aarch64_ad.m4
11965 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11966 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11967 %{
11968 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11969 ins_cost(INSN_COST);
11970 format %{ "add $dst, $src1, $src2, sxth" %}
11971
11972 ins_encode %{
11973 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11974 as_Register($src2$$reg), ext::sxth);
11975 %}
11976 ins_pipe(ialu_reg_reg);
11977 %}
11978
11979 // This pattern is automatically generated from aarch64_ad.m4
11980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11981 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11982 %{
11983 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11984 ins_cost(INSN_COST);
11985 format %{ "add $dst, $src1, $src2, sxtw" %}
11986
11987 ins_encode %{
11988 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11989 as_Register($src2$$reg), ext::sxtw);
11990 %}
11991 ins_pipe(ialu_reg_reg);
11992 %}
11993
11994 // This pattern is automatically generated from aarch64_ad.m4
11995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11996 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11997 %{
11998 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11999 ins_cost(INSN_COST);
12000 format %{ "add $dst, $src1, $src2, sxtb" %}
12001
12002 ins_encode %{
12003 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12004 as_Register($src2$$reg), ext::sxtb);
12005 %}
12006 ins_pipe(ialu_reg_reg);
12007 %}
12008
12009 // This pattern is automatically generated from aarch64_ad.m4
12010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12011 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12012 %{
12013 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12014 ins_cost(INSN_COST);
12015 format %{ "add $dst, $src1, $src2, uxtb" %}
12016
12017 ins_encode %{
12018 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12019 as_Register($src2$$reg), ext::uxtb);
12020 %}
12021 ins_pipe(ialu_reg_reg);
12022 %}
12023
12024 // This pattern is automatically generated from aarch64_ad.m4
12025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12026 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12027 %{
12028 match(Set dst (AddI src1 (AndI src2 mask)));
12029 ins_cost(INSN_COST);
12030 format %{ "addw $dst, $src1, $src2, uxtb" %}
12031
12032 ins_encode %{
12033 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12034 as_Register($src2$$reg), ext::uxtb);
12035 %}
12036 ins_pipe(ialu_reg_reg);
12037 %}
12038
12039 // This pattern is automatically generated from aarch64_ad.m4
12040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12041 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12042 %{
12043 match(Set dst (AddI src1 (AndI src2 mask)));
12044 ins_cost(INSN_COST);
12045 format %{ "addw $dst, $src1, $src2, uxth" %}
12046
12047 ins_encode %{
12048 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12049 as_Register($src2$$reg), ext::uxth);
12050 %}
12051 ins_pipe(ialu_reg_reg);
12052 %}
12053
12054 // This pattern is automatically generated from aarch64_ad.m4
12055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12056 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12057 %{
12058 match(Set dst (AddL src1 (AndL src2 mask)));
12059 ins_cost(INSN_COST);
12060 format %{ "add $dst, $src1, $src2, uxtb" %}
12061
12062 ins_encode %{
12063 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12064 as_Register($src2$$reg), ext::uxtb);
12065 %}
12066 ins_pipe(ialu_reg_reg);
12067 %}
12068
12069 // This pattern is automatically generated from aarch64_ad.m4
12070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12071 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12072 %{
12073 match(Set dst (AddL src1 (AndL src2 mask)));
12074 ins_cost(INSN_COST);
12075 format %{ "add $dst, $src1, $src2, uxth" %}
12076
12077 ins_encode %{
12078 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12079 as_Register($src2$$reg), ext::uxth);
12080 %}
12081 ins_pipe(ialu_reg_reg);
12082 %}
12083
12084 // This pattern is automatically generated from aarch64_ad.m4
12085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12086 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12087 %{
12088 match(Set dst (AddL src1 (AndL src2 mask)));
12089 ins_cost(INSN_COST);
12090 format %{ "add $dst, $src1, $src2, uxtw" %}
12091
12092 ins_encode %{
12093 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12094 as_Register($src2$$reg), ext::uxtw);
12095 %}
12096 ins_pipe(ialu_reg_reg);
12097 %}
12098
12099 // This pattern is automatically generated from aarch64_ad.m4
12100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12101 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12102 %{
12103 match(Set dst (SubI src1 (AndI src2 mask)));
12104 ins_cost(INSN_COST);
12105 format %{ "subw $dst, $src1, $src2, uxtb" %}
12106
12107 ins_encode %{
12108 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12109 as_Register($src2$$reg), ext::uxtb);
12110 %}
12111 ins_pipe(ialu_reg_reg);
12112 %}
12113
12114 // This pattern is automatically generated from aarch64_ad.m4
12115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12116 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12117 %{
12118 match(Set dst (SubI src1 (AndI src2 mask)));
12119 ins_cost(INSN_COST);
12120 format %{ "subw $dst, $src1, $src2, uxth" %}
12121
12122 ins_encode %{
12123 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12124 as_Register($src2$$reg), ext::uxth);
12125 %}
12126 ins_pipe(ialu_reg_reg);
12127 %}
12128
12129 // This pattern is automatically generated from aarch64_ad.m4
12130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12131 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12132 %{
12133 match(Set dst (SubL src1 (AndL src2 mask)));
12134 ins_cost(INSN_COST);
12135 format %{ "sub $dst, $src1, $src2, uxtb" %}
12136
12137 ins_encode %{
12138 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12139 as_Register($src2$$reg), ext::uxtb);
12140 %}
12141 ins_pipe(ialu_reg_reg);
12142 %}
12143
12144 // This pattern is automatically generated from aarch64_ad.m4
12145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12146 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12147 %{
12148 match(Set dst (SubL src1 (AndL src2 mask)));
12149 ins_cost(INSN_COST);
12150 format %{ "sub $dst, $src1, $src2, uxth" %}
12151
12152 ins_encode %{
12153 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12154 as_Register($src2$$reg), ext::uxth);
12155 %}
12156 ins_pipe(ialu_reg_reg);
12157 %}
12158
12159 // This pattern is automatically generated from aarch64_ad.m4
12160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12161 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12162 %{
12163 match(Set dst (SubL src1 (AndL src2 mask)));
12164 ins_cost(INSN_COST);
12165 format %{ "sub $dst, $src1, $src2, uxtw" %}
12166
12167 ins_encode %{
12168 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12169 as_Register($src2$$reg), ext::uxtw);
12170 %}
12171 ins_pipe(ialu_reg_reg);
12172 %}
12173
12174
12175 // This pattern is automatically generated from aarch64_ad.m4
12176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12177 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12178 %{
12179 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12180 ins_cost(1.9 * INSN_COST);
12181 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12182
12183 ins_encode %{
12184 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12185 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12186 %}
12187 ins_pipe(ialu_reg_reg_shift);
12188 %}
12189
12190 // This pattern is automatically generated from aarch64_ad.m4
12191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12192 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12193 %{
12194 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12195 ins_cost(1.9 * INSN_COST);
12196 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12197
12198 ins_encode %{
12199 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12200 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12201 %}
12202 ins_pipe(ialu_reg_reg_shift);
12203 %}
12204
12205 // This pattern is automatically generated from aarch64_ad.m4
12206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12207 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12208 %{
12209 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12210 ins_cost(1.9 * INSN_COST);
12211 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12212
12213 ins_encode %{
12214 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12215 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12216 %}
12217 ins_pipe(ialu_reg_reg_shift);
12218 %}
12219
12220 // This pattern is automatically generated from aarch64_ad.m4
12221 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12222 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12223 %{
12224 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12225 ins_cost(1.9 * INSN_COST);
12226 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12227
12228 ins_encode %{
12229 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12230 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12231 %}
12232 ins_pipe(ialu_reg_reg_shift);
12233 %}
12234
12235 // This pattern is automatically generated from aarch64_ad.m4
12236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12237 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12238 %{
12239 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12240 ins_cost(1.9 * INSN_COST);
12241 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12242
12243 ins_encode %{
12244 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12245 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12246 %}
12247 ins_pipe(ialu_reg_reg_shift);
12248 %}
12249
12250 // This pattern is automatically generated from aarch64_ad.m4
12251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12252 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12253 %{
12254 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12255 ins_cost(1.9 * INSN_COST);
12256 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12257
12258 ins_encode %{
12259 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12260 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12261 %}
12262 ins_pipe(ialu_reg_reg_shift);
12263 %}
12264
12265 // This pattern is automatically generated from aarch64_ad.m4
12266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12267 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12268 %{
12269 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12270 ins_cost(1.9 * INSN_COST);
12271 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12272
12273 ins_encode %{
12274 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12275 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12276 %}
12277 ins_pipe(ialu_reg_reg_shift);
12278 %}
12279
12280 // This pattern is automatically generated from aarch64_ad.m4
12281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12282 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12283 %{
12284 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12285 ins_cost(1.9 * INSN_COST);
12286 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12287
12288 ins_encode %{
12289 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12290 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12291 %}
12292 ins_pipe(ialu_reg_reg_shift);
12293 %}
12294
12295 // This pattern is automatically generated from aarch64_ad.m4
12296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12297 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12298 %{
12299 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12300 ins_cost(1.9 * INSN_COST);
12301 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12302
12303 ins_encode %{
12304 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12305 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12306 %}
12307 ins_pipe(ialu_reg_reg_shift);
12308 %}
12309
12310 // This pattern is automatically generated from aarch64_ad.m4
12311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12312 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12313 %{
12314 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12315 ins_cost(1.9 * INSN_COST);
12316 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12317
12318 ins_encode %{
12319 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12320 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12321 %}
12322 ins_pipe(ialu_reg_reg_shift);
12323 %}
12324
12325 // This pattern is automatically generated from aarch64_ad.m4
12326 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12327 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12328 %{
12329 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12330 ins_cost(1.9 * INSN_COST);
12331 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12332
12333 ins_encode %{
12334 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12335 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12336 %}
12337 ins_pipe(ialu_reg_reg_shift);
12338 %}
12339
12340 // This pattern is automatically generated from aarch64_ad.m4
12341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12342 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12343 %{
12344 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12345 ins_cost(1.9 * INSN_COST);
12346 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12347
12348 ins_encode %{
12349 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12350 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12351 %}
12352 ins_pipe(ialu_reg_reg_shift);
12353 %}
12354
12355 // This pattern is automatically generated from aarch64_ad.m4
12356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12357 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12358 %{
12359 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12360 ins_cost(1.9 * INSN_COST);
12361 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12362
12363 ins_encode %{
12364 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12365 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12366 %}
12367 ins_pipe(ialu_reg_reg_shift);
12368 %}
12369
12370 // This pattern is automatically generated from aarch64_ad.m4
12371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12372 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12373 %{
12374 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12375 ins_cost(1.9 * INSN_COST);
12376 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12377
12378 ins_encode %{
12379 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12380 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12381 %}
12382 ins_pipe(ialu_reg_reg_shift);
12383 %}
12384
12385 // This pattern is automatically generated from aarch64_ad.m4
12386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12387 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12388 %{
12389 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12390 ins_cost(1.9 * INSN_COST);
12391 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12392
12393 ins_encode %{
12394 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12395 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12396 %}
12397 ins_pipe(ialu_reg_reg_shift);
12398 %}
12399
12400 // This pattern is automatically generated from aarch64_ad.m4
12401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12402 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12403 %{
12404 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12405 ins_cost(1.9 * INSN_COST);
12406 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12407
12408 ins_encode %{
12409 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12410 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12411 %}
12412 ins_pipe(ialu_reg_reg_shift);
12413 %}
12414
12415 // This pattern is automatically generated from aarch64_ad.m4
12416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12417 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12418 %{
12419 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12420 ins_cost(1.9 * INSN_COST);
12421 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12422
12423 ins_encode %{
12424 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12425 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12426 %}
12427 ins_pipe(ialu_reg_reg_shift);
12428 %}
12429
12430 // This pattern is automatically generated from aarch64_ad.m4
12431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12432 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12433 %{
12434 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12435 ins_cost(1.9 * INSN_COST);
12436 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12437
12438 ins_encode %{
12439 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12440 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12441 %}
12442 ins_pipe(ialu_reg_reg_shift);
12443 %}
12444
12445 // This pattern is automatically generated from aarch64_ad.m4
12446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12447 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12448 %{
12449 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12450 ins_cost(1.9 * INSN_COST);
12451 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12452
12453 ins_encode %{
12454 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12455 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12456 %}
12457 ins_pipe(ialu_reg_reg_shift);
12458 %}
12459
12460 // This pattern is automatically generated from aarch64_ad.m4
12461 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12462 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12463 %{
12464 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12465 ins_cost(1.9 * INSN_COST);
12466 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12467
12468 ins_encode %{
12469 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12470 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12471 %}
12472 ins_pipe(ialu_reg_reg_shift);
12473 %}
12474
12475 // This pattern is automatically generated from aarch64_ad.m4
12476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12477 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12478 %{
12479 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12480 ins_cost(1.9 * INSN_COST);
12481 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12482
12483 ins_encode %{
12484 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12485 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12486 %}
12487 ins_pipe(ialu_reg_reg_shift);
12488 %}
12489
12490 // This pattern is automatically generated from aarch64_ad.m4
12491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12492 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12493 %{
12494 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12495 ins_cost(1.9 * INSN_COST);
12496 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12497
12498 ins_encode %{
12499 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12500 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12501 %}
12502 ins_pipe(ialu_reg_reg_shift);
12503 %}
12504
12505 // This pattern is automatically generated from aarch64_ad.m4
12506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12507 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12508 %{
12509 effect(DEF dst, USE src1, USE src2, USE cr);
12510 ins_cost(INSN_COST * 2);
12511 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12512
12513 ins_encode %{
12514 __ cselw($dst$$Register,
12515 $src1$$Register,
12516 $src2$$Register,
12517 Assembler::LT);
12518 %}
12519 ins_pipe(icond_reg_reg);
12520 %}
12521
12522 // This pattern is automatically generated from aarch64_ad.m4
12523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12524 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12525 %{
12526 effect(DEF dst, USE src1, USE src2, USE cr);
12527 ins_cost(INSN_COST * 2);
12528 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12529
12530 ins_encode %{
12531 __ cselw($dst$$Register,
12532 $src1$$Register,
12533 $src2$$Register,
12534 Assembler::GT);
12535 %}
12536 ins_pipe(icond_reg_reg);
12537 %}
12538
12539 // This pattern is automatically generated from aarch64_ad.m4
12540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12541 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12542 %{
12543 effect(DEF dst, USE src1, USE cr);
12544 ins_cost(INSN_COST * 2);
12545 format %{ "cselw $dst, $src1, zr lt\t" %}
12546
12547 ins_encode %{
12548 __ cselw($dst$$Register,
12549 $src1$$Register,
12550 zr,
12551 Assembler::LT);
12552 %}
12553 ins_pipe(icond_reg);
12554 %}
12555
12556 // This pattern is automatically generated from aarch64_ad.m4
12557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12558 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12559 %{
12560 effect(DEF dst, USE src1, USE cr);
12561 ins_cost(INSN_COST * 2);
12562 format %{ "cselw $dst, $src1, zr gt\t" %}
12563
12564 ins_encode %{
12565 __ cselw($dst$$Register,
12566 $src1$$Register,
12567 zr,
12568 Assembler::GT);
12569 %}
12570 ins_pipe(icond_reg);
12571 %}
12572
12573 // This pattern is automatically generated from aarch64_ad.m4
12574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12575 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12576 %{
12577 effect(DEF dst, USE src1, USE cr);
12578 ins_cost(INSN_COST * 2);
12579 format %{ "csincw $dst, $src1, zr le\t" %}
12580
12581 ins_encode %{
12582 __ csincw($dst$$Register,
12583 $src1$$Register,
12584 zr,
12585 Assembler::LE);
12586 %}
12587 ins_pipe(icond_reg);
12588 %}
12589
12590 // This pattern is automatically generated from aarch64_ad.m4
12591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12592 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12593 %{
12594 effect(DEF dst, USE src1, USE cr);
12595 ins_cost(INSN_COST * 2);
12596 format %{ "csincw $dst, $src1, zr gt\t" %}
12597
12598 ins_encode %{
12599 __ csincw($dst$$Register,
12600 $src1$$Register,
12601 zr,
12602 Assembler::GT);
12603 %}
12604 ins_pipe(icond_reg);
12605 %}
12606
12607 // This pattern is automatically generated from aarch64_ad.m4
12608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12609 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12610 %{
12611 effect(DEF dst, USE src1, USE cr);
12612 ins_cost(INSN_COST * 2);
12613 format %{ "csinvw $dst, $src1, zr lt\t" %}
12614
12615 ins_encode %{
12616 __ csinvw($dst$$Register,
12617 $src1$$Register,
12618 zr,
12619 Assembler::LT);
12620 %}
12621 ins_pipe(icond_reg);
12622 %}
12623
12624 // This pattern is automatically generated from aarch64_ad.m4
12625 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12626 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12627 %{
12628 effect(DEF dst, USE src1, USE cr);
12629 ins_cost(INSN_COST * 2);
12630 format %{ "csinvw $dst, $src1, zr ge\t" %}
12631
12632 ins_encode %{
12633 __ csinvw($dst$$Register,
12634 $src1$$Register,
12635 zr,
12636 Assembler::GE);
12637 %}
12638 ins_pipe(icond_reg);
12639 %}
12640
12641 // This pattern is automatically generated from aarch64_ad.m4
12642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12643 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12644 %{
12645 match(Set dst (MinI src imm));
12646 ins_cost(INSN_COST * 3);
12647 expand %{
12648 rFlagsReg cr;
12649 compI_reg_imm0(cr, src);
12650 cmovI_reg_imm0_lt(dst, src, cr);
12651 %}
12652 %}
12653
12654 // This pattern is automatically generated from aarch64_ad.m4
12655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12656 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12657 %{
12658 match(Set dst (MinI imm src));
12659 ins_cost(INSN_COST * 3);
12660 expand %{
12661 rFlagsReg cr;
12662 compI_reg_imm0(cr, src);
12663 cmovI_reg_imm0_lt(dst, src, cr);
12664 %}
12665 %}
12666
12667 // This pattern is automatically generated from aarch64_ad.m4
12668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12669 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12670 %{
12671 match(Set dst (MinI src imm));
12672 ins_cost(INSN_COST * 3);
12673 expand %{
12674 rFlagsReg cr;
12675 compI_reg_imm0(cr, src);
12676 cmovI_reg_imm1_le(dst, src, cr);
12677 %}
12678 %}
12679
12680 // This pattern is automatically generated from aarch64_ad.m4
12681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12682 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12683 %{
12684 match(Set dst (MinI imm src));
12685 ins_cost(INSN_COST * 3);
12686 expand %{
12687 rFlagsReg cr;
12688 compI_reg_imm0(cr, src);
12689 cmovI_reg_imm1_le(dst, src, cr);
12690 %}
12691 %}
12692
12693 // This pattern is automatically generated from aarch64_ad.m4
12694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12695 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12696 %{
12697 match(Set dst (MinI src imm));
12698 ins_cost(INSN_COST * 3);
12699 expand %{
12700 rFlagsReg cr;
12701 compI_reg_imm0(cr, src);
12702 cmovI_reg_immM1_lt(dst, src, cr);
12703 %}
12704 %}
12705
12706 // This pattern is automatically generated from aarch64_ad.m4
12707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12708 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12709 %{
12710 match(Set dst (MinI imm src));
12711 ins_cost(INSN_COST * 3);
12712 expand %{
12713 rFlagsReg cr;
12714 compI_reg_imm0(cr, src);
12715 cmovI_reg_immM1_lt(dst, src, cr);
12716 %}
12717 %}
12718
12719 // This pattern is automatically generated from aarch64_ad.m4
12720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12721 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12722 %{
12723 match(Set dst (MaxI src imm));
12724 ins_cost(INSN_COST * 3);
12725 expand %{
12726 rFlagsReg cr;
12727 compI_reg_imm0(cr, src);
12728 cmovI_reg_imm0_gt(dst, src, cr);
12729 %}
12730 %}
12731
12732 // This pattern is automatically generated from aarch64_ad.m4
12733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12734 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12735 %{
12736 match(Set dst (MaxI imm src));
12737 ins_cost(INSN_COST * 3);
12738 expand %{
12739 rFlagsReg cr;
12740 compI_reg_imm0(cr, src);
12741 cmovI_reg_imm0_gt(dst, src, cr);
12742 %}
12743 %}
12744
12745 // This pattern is automatically generated from aarch64_ad.m4
12746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12747 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12748 %{
12749 match(Set dst (MaxI src imm));
12750 ins_cost(INSN_COST * 3);
12751 expand %{
12752 rFlagsReg cr;
12753 compI_reg_imm0(cr, src);
12754 cmovI_reg_imm1_gt(dst, src, cr);
12755 %}
12756 %}
12757
12758 // This pattern is automatically generated from aarch64_ad.m4
12759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12760 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12761 %{
12762 match(Set dst (MaxI imm src));
12763 ins_cost(INSN_COST * 3);
12764 expand %{
12765 rFlagsReg cr;
12766 compI_reg_imm0(cr, src);
12767 cmovI_reg_imm1_gt(dst, src, cr);
12768 %}
12769 %}
12770
12771 // This pattern is automatically generated from aarch64_ad.m4
12772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12773 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12774 %{
12775 match(Set dst (MaxI src imm));
12776 ins_cost(INSN_COST * 3);
12777 expand %{
12778 rFlagsReg cr;
12779 compI_reg_imm0(cr, src);
12780 cmovI_reg_immM1_ge(dst, src, cr);
12781 %}
12782 %}
12783
12784 // This pattern is automatically generated from aarch64_ad.m4
12785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12786 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12787 %{
12788 match(Set dst (MaxI imm src));
12789 ins_cost(INSN_COST * 3);
12790 expand %{
12791 rFlagsReg cr;
12792 compI_reg_imm0(cr, src);
12793 cmovI_reg_immM1_ge(dst, src, cr);
12794 %}
12795 %}
12796
12797 // This pattern is automatically generated from aarch64_ad.m4
12798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12799 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12800 %{
12801 match(Set dst (ReverseI src));
12802 ins_cost(INSN_COST);
12803 format %{ "rbitw $dst, $src" %}
12804 ins_encode %{
12805 __ rbitw($dst$$Register, $src$$Register);
12806 %}
12807 ins_pipe(ialu_reg);
12808 %}
12809
12810 // This pattern is automatically generated from aarch64_ad.m4
12811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12812 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12813 %{
12814 match(Set dst (ReverseL src));
12815 ins_cost(INSN_COST);
12816 format %{ "rbit $dst, $src" %}
12817 ins_encode %{
12818 __ rbit($dst$$Register, $src$$Register);
12819 %}
12820 ins_pipe(ialu_reg);
12821 %}
12822
12823
12824 // END This section of the file is automatically generated. Do not edit --------------
12825
12826
12827 // ============================================================================
12828 // Floating Point Arithmetic Instructions
12829
12830 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12831 match(Set dst (AddHF src1 src2));
12832 format %{ "faddh $dst, $src1, $src2" %}
12833 ins_encode %{
12834 __ faddh($dst$$FloatRegister,
12835 $src1$$FloatRegister,
12836 $src2$$FloatRegister);
12837 %}
12838 ins_pipe(fp_dop_reg_reg_s);
12839 %}
12840
12841 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12842 match(Set dst (AddF src1 src2));
12843
12844 ins_cost(INSN_COST * 5);
12845 format %{ "fadds $dst, $src1, $src2" %}
12846
12847 ins_encode %{
12848 __ fadds(as_FloatRegister($dst$$reg),
12849 as_FloatRegister($src1$$reg),
12850 as_FloatRegister($src2$$reg));
12851 %}
12852
12853 ins_pipe(fp_dop_reg_reg_s);
12854 %}
12855
12856 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12857 match(Set dst (AddD src1 src2));
12858
12859 ins_cost(INSN_COST * 5);
12860 format %{ "faddd $dst, $src1, $src2" %}
12861
12862 ins_encode %{
12863 __ faddd(as_FloatRegister($dst$$reg),
12864 as_FloatRegister($src1$$reg),
12865 as_FloatRegister($src2$$reg));
12866 %}
12867
12868 ins_pipe(fp_dop_reg_reg_d);
12869 %}
12870
12871 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12872 match(Set dst (SubHF src1 src2));
12873 format %{ "fsubh $dst, $src1, $src2" %}
12874 ins_encode %{
12875 __ fsubh($dst$$FloatRegister,
12876 $src1$$FloatRegister,
12877 $src2$$FloatRegister);
12878 %}
12879 ins_pipe(fp_dop_reg_reg_s);
12880 %}
12881
12882 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12883 match(Set dst (SubF src1 src2));
12884
12885 ins_cost(INSN_COST * 5);
12886 format %{ "fsubs $dst, $src1, $src2" %}
12887
12888 ins_encode %{
12889 __ fsubs(as_FloatRegister($dst$$reg),
12890 as_FloatRegister($src1$$reg),
12891 as_FloatRegister($src2$$reg));
12892 %}
12893
12894 ins_pipe(fp_dop_reg_reg_s);
12895 %}
12896
12897 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12898 match(Set dst (SubD src1 src2));
12899
12900 ins_cost(INSN_COST * 5);
12901 format %{ "fsubd $dst, $src1, $src2" %}
12902
12903 ins_encode %{
12904 __ fsubd(as_FloatRegister($dst$$reg),
12905 as_FloatRegister($src1$$reg),
12906 as_FloatRegister($src2$$reg));
12907 %}
12908
12909 ins_pipe(fp_dop_reg_reg_d);
12910 %}
12911
12912 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12913 match(Set dst (MulHF src1 src2));
12914 format %{ "fmulh $dst, $src1, $src2" %}
12915 ins_encode %{
12916 __ fmulh($dst$$FloatRegister,
12917 $src1$$FloatRegister,
12918 $src2$$FloatRegister);
12919 %}
12920 ins_pipe(fp_dop_reg_reg_s);
12921 %}
12922
12923 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12924 match(Set dst (MulF src1 src2));
12925
12926 ins_cost(INSN_COST * 6);
12927 format %{ "fmuls $dst, $src1, $src2" %}
12928
12929 ins_encode %{
12930 __ fmuls(as_FloatRegister($dst$$reg),
12931 as_FloatRegister($src1$$reg),
12932 as_FloatRegister($src2$$reg));
12933 %}
12934
12935 ins_pipe(fp_dop_reg_reg_s);
12936 %}
12937
12938 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12939 match(Set dst (MulD src1 src2));
12940
12941 ins_cost(INSN_COST * 6);
12942 format %{ "fmuld $dst, $src1, $src2" %}
12943
12944 ins_encode %{
12945 __ fmuld(as_FloatRegister($dst$$reg),
12946 as_FloatRegister($src1$$reg),
12947 as_FloatRegister($src2$$reg));
12948 %}
12949
12950 ins_pipe(fp_dop_reg_reg_d);
12951 %}
12952
12953 // src1 * src2 + src3 (half-precision float)
12954 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12955 match(Set dst (FmaHF src3 (Binary src1 src2)));
12956 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12957 ins_encode %{
12958 assert(UseFMA, "Needs FMA instructions support.");
12959 __ fmaddh($dst$$FloatRegister,
12960 $src1$$FloatRegister,
12961 $src2$$FloatRegister,
12962 $src3$$FloatRegister);
12963 %}
12964 ins_pipe(pipe_class_default);
12965 %}
12966
12967 // src1 * src2 + src3
12968 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12969 match(Set dst (FmaF src3 (Binary src1 src2)));
12970
12971 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12972
12973 ins_encode %{
12974 assert(UseFMA, "Needs FMA instructions support.");
12975 __ fmadds(as_FloatRegister($dst$$reg),
12976 as_FloatRegister($src1$$reg),
12977 as_FloatRegister($src2$$reg),
12978 as_FloatRegister($src3$$reg));
12979 %}
12980
12981 ins_pipe(pipe_class_default);
12982 %}
12983
12984 // src1 * src2 + src3
12985 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12986 match(Set dst (FmaD src3 (Binary src1 src2)));
12987
12988 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12989
12990 ins_encode %{
12991 assert(UseFMA, "Needs FMA instructions support.");
12992 __ fmaddd(as_FloatRegister($dst$$reg),
12993 as_FloatRegister($src1$$reg),
12994 as_FloatRegister($src2$$reg),
12995 as_FloatRegister($src3$$reg));
12996 %}
12997
12998 ins_pipe(pipe_class_default);
12999 %}
13000
13001 // src1 * (-src2) + src3
13002 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13003 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13004 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13005
13006 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13007
13008 ins_encode %{
13009 assert(UseFMA, "Needs FMA instructions support.");
13010 __ fmsubs(as_FloatRegister($dst$$reg),
13011 as_FloatRegister($src1$$reg),
13012 as_FloatRegister($src2$$reg),
13013 as_FloatRegister($src3$$reg));
13014 %}
13015
13016 ins_pipe(pipe_class_default);
13017 %}
13018
13019 // src1 * (-src2) + src3
13020 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13021 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13022 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13023
13024 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13025
13026 ins_encode %{
13027 assert(UseFMA, "Needs FMA instructions support.");
13028 __ fmsubd(as_FloatRegister($dst$$reg),
13029 as_FloatRegister($src1$$reg),
13030 as_FloatRegister($src2$$reg),
13031 as_FloatRegister($src3$$reg));
13032 %}
13033
13034 ins_pipe(pipe_class_default);
13035 %}
13036
13037 // src1 * (-src2) - src3
13038 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13039 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13040 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13041
13042 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13043
13044 ins_encode %{
13045 assert(UseFMA, "Needs FMA instructions support.");
13046 __ fnmadds(as_FloatRegister($dst$$reg),
13047 as_FloatRegister($src1$$reg),
13048 as_FloatRegister($src2$$reg),
13049 as_FloatRegister($src3$$reg));
13050 %}
13051
13052 ins_pipe(pipe_class_default);
13053 %}
13054
13055 // src1 * (-src2) - src3
13056 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13057 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13058 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13059
13060 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13061
13062 ins_encode %{
13063 assert(UseFMA, "Needs FMA instructions support.");
13064 __ fnmaddd(as_FloatRegister($dst$$reg),
13065 as_FloatRegister($src1$$reg),
13066 as_FloatRegister($src2$$reg),
13067 as_FloatRegister($src3$$reg));
13068 %}
13069
13070 ins_pipe(pipe_class_default);
13071 %}
13072
13073 // src1 * src2 - src3
13074 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13075 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13076
13077 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13078
13079 ins_encode %{
13080 assert(UseFMA, "Needs FMA instructions support.");
13081 __ fnmsubs(as_FloatRegister($dst$$reg),
13082 as_FloatRegister($src1$$reg),
13083 as_FloatRegister($src2$$reg),
13084 as_FloatRegister($src3$$reg));
13085 %}
13086
13087 ins_pipe(pipe_class_default);
13088 %}
13089
13090 // src1 * src2 - src3
13091 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13092 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13093
13094 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13095
13096 ins_encode %{
13097 assert(UseFMA, "Needs FMA instructions support.");
13098 // n.b. insn name should be fnmsubd
13099 __ fnmsub(as_FloatRegister($dst$$reg),
13100 as_FloatRegister($src1$$reg),
13101 as_FloatRegister($src2$$reg),
13102 as_FloatRegister($src3$$reg));
13103 %}
13104
13105 ins_pipe(pipe_class_default);
13106 %}
13107
13108 // Math.max(HH)H (half-precision float)
13109 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13110 match(Set dst (MaxHF src1 src2));
13111 format %{ "fmaxh $dst, $src1, $src2" %}
13112 ins_encode %{
13113 __ fmaxh($dst$$FloatRegister,
13114 $src1$$FloatRegister,
13115 $src2$$FloatRegister);
13116 %}
13117 ins_pipe(fp_dop_reg_reg_s);
13118 %}
13119
13120 // Math.min(HH)H (half-precision float)
13121 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13122 match(Set dst (MinHF src1 src2));
13123 format %{ "fminh $dst, $src1, $src2" %}
13124 ins_encode %{
13125 __ fminh($dst$$FloatRegister,
13126 $src1$$FloatRegister,
13127 $src2$$FloatRegister);
13128 %}
13129 ins_pipe(fp_dop_reg_reg_s);
13130 %}
13131
13132 // Math.max(FF)F
13133 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13134 match(Set dst (MaxF src1 src2));
13135
13136 format %{ "fmaxs $dst, $src1, $src2" %}
13137 ins_encode %{
13138 __ fmaxs(as_FloatRegister($dst$$reg),
13139 as_FloatRegister($src1$$reg),
13140 as_FloatRegister($src2$$reg));
13141 %}
13142
13143 ins_pipe(fp_dop_reg_reg_s);
13144 %}
13145
13146 // Math.min(FF)F
13147 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13148 match(Set dst (MinF src1 src2));
13149
13150 format %{ "fmins $dst, $src1, $src2" %}
13151 ins_encode %{
13152 __ fmins(as_FloatRegister($dst$$reg),
13153 as_FloatRegister($src1$$reg),
13154 as_FloatRegister($src2$$reg));
13155 %}
13156
13157 ins_pipe(fp_dop_reg_reg_s);
13158 %}
13159
13160 // Math.max(DD)D
13161 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13162 match(Set dst (MaxD src1 src2));
13163
13164 format %{ "fmaxd $dst, $src1, $src2" %}
13165 ins_encode %{
13166 __ fmaxd(as_FloatRegister($dst$$reg),
13167 as_FloatRegister($src1$$reg),
13168 as_FloatRegister($src2$$reg));
13169 %}
13170
13171 ins_pipe(fp_dop_reg_reg_d);
13172 %}
13173
13174 // Math.min(DD)D
13175 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13176 match(Set dst (MinD src1 src2));
13177
13178 format %{ "fmind $dst, $src1, $src2" %}
13179 ins_encode %{
13180 __ fmind(as_FloatRegister($dst$$reg),
13181 as_FloatRegister($src1$$reg),
13182 as_FloatRegister($src2$$reg));
13183 %}
13184
13185 ins_pipe(fp_dop_reg_reg_d);
13186 %}
13187
13188 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13189 match(Set dst (DivHF src1 src2));
13190 format %{ "fdivh $dst, $src1, $src2" %}
13191 ins_encode %{
13192 __ fdivh($dst$$FloatRegister,
13193 $src1$$FloatRegister,
13194 $src2$$FloatRegister);
13195 %}
13196 ins_pipe(fp_div_s);
13197 %}
13198
13199 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13200 match(Set dst (DivF src1 src2));
13201
13202 ins_cost(INSN_COST * 18);
13203 format %{ "fdivs $dst, $src1, $src2" %}
13204
13205 ins_encode %{
13206 __ fdivs(as_FloatRegister($dst$$reg),
13207 as_FloatRegister($src1$$reg),
13208 as_FloatRegister($src2$$reg));
13209 %}
13210
13211 ins_pipe(fp_div_s);
13212 %}
13213
13214 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13215 match(Set dst (DivD src1 src2));
13216
13217 ins_cost(INSN_COST * 32);
13218 format %{ "fdivd $dst, $src1, $src2" %}
13219
13220 ins_encode %{
13221 __ fdivd(as_FloatRegister($dst$$reg),
13222 as_FloatRegister($src1$$reg),
13223 as_FloatRegister($src2$$reg));
13224 %}
13225
13226 ins_pipe(fp_div_d);
13227 %}
13228
13229 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13230 match(Set dst (NegF src));
13231
13232 ins_cost(INSN_COST * 3);
13233 format %{ "fneg $dst, $src" %}
13234
13235 ins_encode %{
13236 __ fnegs(as_FloatRegister($dst$$reg),
13237 as_FloatRegister($src$$reg));
13238 %}
13239
13240 ins_pipe(fp_uop_s);
13241 %}
13242
13243 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13244 match(Set dst (NegD src));
13245
13246 ins_cost(INSN_COST * 3);
13247 format %{ "fnegd $dst, $src" %}
13248
13249 ins_encode %{
13250 __ fnegd(as_FloatRegister($dst$$reg),
13251 as_FloatRegister($src$$reg));
13252 %}
13253
13254 ins_pipe(fp_uop_d);
13255 %}
13256
13257 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13258 %{
13259 match(Set dst (AbsI src));
13260
13261 effect(KILL cr);
13262 ins_cost(INSN_COST * 2);
13263 format %{ "cmpw $src, zr\n\t"
13264 "cnegw $dst, $src, Assembler::LT\t# int abs"
13265 %}
13266
13267 ins_encode %{
13268 __ cmpw(as_Register($src$$reg), zr);
13269 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13270 %}
13271 ins_pipe(pipe_class_default);
13272 %}
13273
13274 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13275 %{
13276 match(Set dst (AbsL src));
13277
13278 effect(KILL cr);
13279 ins_cost(INSN_COST * 2);
13280 format %{ "cmp $src, zr\n\t"
13281 "cneg $dst, $src, Assembler::LT\t# long abs"
13282 %}
13283
13284 ins_encode %{
13285 __ cmp(as_Register($src$$reg), zr);
13286 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13287 %}
13288 ins_pipe(pipe_class_default);
13289 %}
13290
13291 instruct absF_reg(vRegF dst, vRegF src) %{
13292 match(Set dst (AbsF src));
13293
13294 ins_cost(INSN_COST * 3);
13295 format %{ "fabss $dst, $src" %}
13296 ins_encode %{
13297 __ fabss(as_FloatRegister($dst$$reg),
13298 as_FloatRegister($src$$reg));
13299 %}
13300
13301 ins_pipe(fp_uop_s);
13302 %}
13303
13304 instruct absD_reg(vRegD dst, vRegD src) %{
13305 match(Set dst (AbsD src));
13306
13307 ins_cost(INSN_COST * 3);
13308 format %{ "fabsd $dst, $src" %}
13309 ins_encode %{
13310 __ fabsd(as_FloatRegister($dst$$reg),
13311 as_FloatRegister($src$$reg));
13312 %}
13313
13314 ins_pipe(fp_uop_d);
13315 %}
13316
13317 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13318 match(Set dst (AbsF (SubF src1 src2)));
13319
13320 ins_cost(INSN_COST * 3);
13321 format %{ "fabds $dst, $src1, $src2" %}
13322 ins_encode %{
13323 __ fabds(as_FloatRegister($dst$$reg),
13324 as_FloatRegister($src1$$reg),
13325 as_FloatRegister($src2$$reg));
13326 %}
13327
13328 ins_pipe(fp_uop_s);
13329 %}
13330
13331 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13332 match(Set dst (AbsD (SubD src1 src2)));
13333
13334 ins_cost(INSN_COST * 3);
13335 format %{ "fabdd $dst, $src1, $src2" %}
13336 ins_encode %{
13337 __ fabdd(as_FloatRegister($dst$$reg),
13338 as_FloatRegister($src1$$reg),
13339 as_FloatRegister($src2$$reg));
13340 %}
13341
13342 ins_pipe(fp_uop_d);
13343 %}
13344
13345 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13346 match(Set dst (SqrtD src));
13347
13348 ins_cost(INSN_COST * 50);
13349 format %{ "fsqrtd $dst, $src" %}
13350 ins_encode %{
13351 __ fsqrtd(as_FloatRegister($dst$$reg),
13352 as_FloatRegister($src$$reg));
13353 %}
13354
13355 ins_pipe(fp_div_s);
13356 %}
13357
13358 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13359 match(Set dst (SqrtF src));
13360
13361 ins_cost(INSN_COST * 50);
13362 format %{ "fsqrts $dst, $src" %}
13363 ins_encode %{
13364 __ fsqrts(as_FloatRegister($dst$$reg),
13365 as_FloatRegister($src$$reg));
13366 %}
13367
13368 ins_pipe(fp_div_d);
13369 %}
13370
13371 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13372 match(Set dst (SqrtHF src));
13373 format %{ "fsqrth $dst, $src" %}
13374 ins_encode %{
13375 __ fsqrth($dst$$FloatRegister,
13376 $src$$FloatRegister);
13377 %}
13378 ins_pipe(fp_div_s);
13379 %}
13380
13381 // Math.rint, floor, ceil
13382 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13383 match(Set dst (RoundDoubleMode src rmode));
13384 format %{ "frint $dst, $src, $rmode" %}
13385 ins_encode %{
13386 switch ($rmode$$constant) {
13387 case RoundDoubleModeNode::rmode_rint:
13388 __ frintnd(as_FloatRegister($dst$$reg),
13389 as_FloatRegister($src$$reg));
13390 break;
13391 case RoundDoubleModeNode::rmode_floor:
13392 __ frintmd(as_FloatRegister($dst$$reg),
13393 as_FloatRegister($src$$reg));
13394 break;
13395 case RoundDoubleModeNode::rmode_ceil:
13396 __ frintpd(as_FloatRegister($dst$$reg),
13397 as_FloatRegister($src$$reg));
13398 break;
13399 }
13400 %}
13401 ins_pipe(fp_uop_d);
13402 %}
13403
13404 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13405 match(Set dst (CopySignD src1 (Binary src2 zero)));
13406 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13407 format %{ "CopySignD $dst $src1 $src2" %}
13408 ins_encode %{
13409 FloatRegister dst = as_FloatRegister($dst$$reg),
13410 src1 = as_FloatRegister($src1$$reg),
13411 src2 = as_FloatRegister($src2$$reg),
13412 zero = as_FloatRegister($zero$$reg);
13413 __ fnegd(dst, zero);
13414 __ bsl(dst, __ T8B, src2, src1);
13415 %}
13416 ins_pipe(fp_uop_d);
13417 %}
13418
13419 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13420 match(Set dst (CopySignF src1 src2));
13421 effect(TEMP_DEF dst, USE src1, USE src2);
13422 format %{ "CopySignF $dst $src1 $src2" %}
13423 ins_encode %{
13424 FloatRegister dst = as_FloatRegister($dst$$reg),
13425 src1 = as_FloatRegister($src1$$reg),
13426 src2 = as_FloatRegister($src2$$reg);
13427 __ movi(dst, __ T2S, 0x80, 24);
13428 __ bsl(dst, __ T8B, src2, src1);
13429 %}
13430 ins_pipe(fp_uop_d);
13431 %}
13432
13433 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13434 match(Set dst (SignumD src (Binary zero one)));
13435 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13436 format %{ "signumD $dst, $src" %}
13437 ins_encode %{
13438 FloatRegister src = as_FloatRegister($src$$reg),
13439 dst = as_FloatRegister($dst$$reg),
13440 zero = as_FloatRegister($zero$$reg),
13441 one = as_FloatRegister($one$$reg);
13442 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13443 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13444 // Bit selection instruction gets bit from "one" for each enabled bit in
13445 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13446 // NaN the whole "src" will be copied because "dst" is zero. For all other
13447 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13448 // from "src", and all other bits are copied from 1.0.
13449 __ bsl(dst, __ T8B, one, src);
13450 %}
13451 ins_pipe(fp_uop_d);
13452 %}
13453
13454 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13455 match(Set dst (SignumF src (Binary zero one)));
13456 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13457 format %{ "signumF $dst, $src" %}
13458 ins_encode %{
13459 FloatRegister src = as_FloatRegister($src$$reg),
13460 dst = as_FloatRegister($dst$$reg),
13461 zero = as_FloatRegister($zero$$reg),
13462 one = as_FloatRegister($one$$reg);
13463 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13464 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13465 // Bit selection instruction gets bit from "one" for each enabled bit in
13466 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13467 // NaN the whole "src" will be copied because "dst" is zero. For all other
13468 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13469 // from "src", and all other bits are copied from 1.0.
13470 __ bsl(dst, __ T8B, one, src);
13471 %}
13472 ins_pipe(fp_uop_d);
13473 %}
13474
13475 instruct onspinwait() %{
13476 match(OnSpinWait);
13477 ins_cost(INSN_COST);
13478
13479 format %{ "onspinwait" %}
13480
13481 ins_encode %{
13482 __ spin_wait();
13483 %}
13484 ins_pipe(pipe_class_empty);
13485 %}
13486
13487 // ============================================================================
13488 // Logical Instructions
13489
13490 // Integer Logical Instructions
13491
13492 // And Instructions
13493
13494
13495 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13496 match(Set dst (AndI src1 src2));
13497
13498 format %{ "andw $dst, $src1, $src2\t# int" %}
13499
13500 ins_cost(INSN_COST);
13501 ins_encode %{
13502 __ andw(as_Register($dst$$reg),
13503 as_Register($src1$$reg),
13504 as_Register($src2$$reg));
13505 %}
13506
13507 ins_pipe(ialu_reg_reg);
13508 %}
13509
13510 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13511 match(Set dst (AndI src1 src2));
13512
13513 format %{ "andsw $dst, $src1, $src2\t# int" %}
13514
13515 ins_cost(INSN_COST);
13516 ins_encode %{
13517 __ andw(as_Register($dst$$reg),
13518 as_Register($src1$$reg),
13519 (uint64_t)($src2$$constant));
13520 %}
13521
13522 ins_pipe(ialu_reg_imm);
13523 %}
13524
13525 // Or Instructions
13526
13527 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13528 match(Set dst (OrI src1 src2));
13529
13530 format %{ "orrw $dst, $src1, $src2\t# int" %}
13531
13532 ins_cost(INSN_COST);
13533 ins_encode %{
13534 __ orrw(as_Register($dst$$reg),
13535 as_Register($src1$$reg),
13536 as_Register($src2$$reg));
13537 %}
13538
13539 ins_pipe(ialu_reg_reg);
13540 %}
13541
13542 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13543 match(Set dst (OrI src1 src2));
13544
13545 format %{ "orrw $dst, $src1, $src2\t# int" %}
13546
13547 ins_cost(INSN_COST);
13548 ins_encode %{
13549 __ orrw(as_Register($dst$$reg),
13550 as_Register($src1$$reg),
13551 (uint64_t)($src2$$constant));
13552 %}
13553
13554 ins_pipe(ialu_reg_imm);
13555 %}
13556
13557 // Xor Instructions
13558
13559 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13560 match(Set dst (XorI src1 src2));
13561
13562 format %{ "eorw $dst, $src1, $src2\t# int" %}
13563
13564 ins_cost(INSN_COST);
13565 ins_encode %{
13566 __ eorw(as_Register($dst$$reg),
13567 as_Register($src1$$reg),
13568 as_Register($src2$$reg));
13569 %}
13570
13571 ins_pipe(ialu_reg_reg);
13572 %}
13573
13574 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13575 match(Set dst (XorI src1 src2));
13576
13577 format %{ "eorw $dst, $src1, $src2\t# int" %}
13578
13579 ins_cost(INSN_COST);
13580 ins_encode %{
13581 __ eorw(as_Register($dst$$reg),
13582 as_Register($src1$$reg),
13583 (uint64_t)($src2$$constant));
13584 %}
13585
13586 ins_pipe(ialu_reg_imm);
13587 %}
13588
13589 // Long Logical Instructions
13590 // TODO
13591
13592 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13593 match(Set dst (AndL src1 src2));
13594
13595 format %{ "and $dst, $src1, $src2\t# int" %}
13596
13597 ins_cost(INSN_COST);
13598 ins_encode %{
13599 __ andr(as_Register($dst$$reg),
13600 as_Register($src1$$reg),
13601 as_Register($src2$$reg));
13602 %}
13603
13604 ins_pipe(ialu_reg_reg);
13605 %}
13606
13607 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13608 match(Set dst (AndL src1 src2));
13609
13610 format %{ "and $dst, $src1, $src2\t# int" %}
13611
13612 ins_cost(INSN_COST);
13613 ins_encode %{
13614 __ andr(as_Register($dst$$reg),
13615 as_Register($src1$$reg),
13616 (uint64_t)($src2$$constant));
13617 %}
13618
13619 ins_pipe(ialu_reg_imm);
13620 %}
13621
13622 // Or Instructions
13623
13624 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13625 match(Set dst (OrL src1 src2));
13626
13627 format %{ "orr $dst, $src1, $src2\t# int" %}
13628
13629 ins_cost(INSN_COST);
13630 ins_encode %{
13631 __ orr(as_Register($dst$$reg),
13632 as_Register($src1$$reg),
13633 as_Register($src2$$reg));
13634 %}
13635
13636 ins_pipe(ialu_reg_reg);
13637 %}
13638
13639 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13640 match(Set dst (OrL src1 src2));
13641
13642 format %{ "orr $dst, $src1, $src2\t# int" %}
13643
13644 ins_cost(INSN_COST);
13645 ins_encode %{
13646 __ orr(as_Register($dst$$reg),
13647 as_Register($src1$$reg),
13648 (uint64_t)($src2$$constant));
13649 %}
13650
13651 ins_pipe(ialu_reg_imm);
13652 %}
13653
13654 // Xor Instructions
13655
13656 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13657 match(Set dst (XorL src1 src2));
13658
13659 format %{ "eor $dst, $src1, $src2\t# int" %}
13660
13661 ins_cost(INSN_COST);
13662 ins_encode %{
13663 __ eor(as_Register($dst$$reg),
13664 as_Register($src1$$reg),
13665 as_Register($src2$$reg));
13666 %}
13667
13668 ins_pipe(ialu_reg_reg);
13669 %}
13670
13671 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13672 match(Set dst (XorL src1 src2));
13673
13674 ins_cost(INSN_COST);
13675 format %{ "eor $dst, $src1, $src2\t# int" %}
13676
13677 ins_encode %{
13678 __ eor(as_Register($dst$$reg),
13679 as_Register($src1$$reg),
13680 (uint64_t)($src2$$constant));
13681 %}
13682
13683 ins_pipe(ialu_reg_imm);
13684 %}
13685
13686 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13687 %{
13688 match(Set dst (ConvI2L src));
13689
13690 ins_cost(INSN_COST);
13691 format %{ "sxtw $dst, $src\t# i2l" %}
13692 ins_encode %{
13693 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13694 %}
13695 ins_pipe(ialu_reg_shift);
13696 %}
13697
13698 // this pattern occurs in bigmath arithmetic
13699 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13700 %{
13701 match(Set dst (AndL (ConvI2L src) mask));
13702
13703 ins_cost(INSN_COST);
13704 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13705 ins_encode %{
13706 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13707 %}
13708
13709 ins_pipe(ialu_reg_shift);
13710 %}
13711
13712 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13713 match(Set dst (ConvL2I src));
13714
13715 ins_cost(INSN_COST);
13716 format %{ "movw $dst, $src \t// l2i" %}
13717
13718 ins_encode %{
13719 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13720 %}
13721
13722 ins_pipe(ialu_reg);
13723 %}
13724
13725 instruct convD2F_reg(vRegF dst, vRegD src) %{
13726 match(Set dst (ConvD2F src));
13727
13728 ins_cost(INSN_COST * 5);
13729 format %{ "fcvtd $dst, $src \t// d2f" %}
13730
13731 ins_encode %{
13732 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13733 %}
13734
13735 ins_pipe(fp_d2f);
13736 %}
13737
13738 instruct convF2D_reg(vRegD dst, vRegF src) %{
13739 match(Set dst (ConvF2D src));
13740
13741 ins_cost(INSN_COST * 5);
13742 format %{ "fcvts $dst, $src \t// f2d" %}
13743
13744 ins_encode %{
13745 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13746 %}
13747
13748 ins_pipe(fp_f2d);
13749 %}
13750
13751 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13752 match(Set dst (ConvF2I src));
13753
13754 ins_cost(INSN_COST * 5);
13755 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13756
13757 ins_encode %{
13758 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13759 %}
13760
13761 ins_pipe(fp_f2i);
13762 %}
13763
13764 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13765 match(Set dst (ConvF2L src));
13766
13767 ins_cost(INSN_COST * 5);
13768 format %{ "fcvtzs $dst, $src \t// f2l" %}
13769
13770 ins_encode %{
13771 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13772 %}
13773
13774 ins_pipe(fp_f2l);
13775 %}
13776
13777 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13778 match(Set dst (ConvF2HF src));
13779 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13780 "smov $dst, $tmp\t# move result from $tmp to $dst"
13781 %}
13782 effect(TEMP tmp);
13783 ins_encode %{
13784 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13785 %}
13786 ins_pipe(pipe_slow);
13787 %}
13788
13789 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13790 match(Set dst (ConvHF2F src));
13791 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13792 "fcvt $dst, $tmp\t# convert half to single precision"
13793 %}
13794 effect(TEMP tmp);
13795 ins_encode %{
13796 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13797 %}
13798 ins_pipe(pipe_slow);
13799 %}
13800
13801 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13802 match(Set dst (ConvI2F src));
13803
13804 ins_cost(INSN_COST * 5);
13805 format %{ "scvtfws $dst, $src \t// i2f" %}
13806
13807 ins_encode %{
13808 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13809 %}
13810
13811 ins_pipe(fp_i2f);
13812 %}
13813
13814 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13815 match(Set dst (ConvL2F src));
13816
13817 ins_cost(INSN_COST * 5);
13818 format %{ "scvtfs $dst, $src \t// l2f" %}
13819
13820 ins_encode %{
13821 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13822 %}
13823
13824 ins_pipe(fp_l2f);
13825 %}
13826
13827 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13828 match(Set dst (ConvD2I src));
13829
13830 ins_cost(INSN_COST * 5);
13831 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13832
13833 ins_encode %{
13834 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13835 %}
13836
13837 ins_pipe(fp_d2i);
13838 %}
13839
13840 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13841 match(Set dst (ConvD2L src));
13842
13843 ins_cost(INSN_COST * 5);
13844 format %{ "fcvtzd $dst, $src \t// d2l" %}
13845
13846 ins_encode %{
13847 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13848 %}
13849
13850 ins_pipe(fp_d2l);
13851 %}
13852
13853 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13854 match(Set dst (ConvI2D src));
13855
13856 ins_cost(INSN_COST * 5);
13857 format %{ "scvtfwd $dst, $src \t// i2d" %}
13858
13859 ins_encode %{
13860 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13861 %}
13862
13863 ins_pipe(fp_i2d);
13864 %}
13865
13866 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13867 match(Set dst (ConvL2D src));
13868
13869 ins_cost(INSN_COST * 5);
13870 format %{ "scvtfd $dst, $src \t// l2d" %}
13871
13872 ins_encode %{
13873 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13874 %}
13875
13876 ins_pipe(fp_l2d);
13877 %}
13878
13879 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13880 %{
13881 match(Set dst (RoundD src));
13882 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13883 format %{ "java_round_double $dst,$src"%}
13884 ins_encode %{
13885 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13886 as_FloatRegister($ftmp$$reg));
13887 %}
13888 ins_pipe(pipe_slow);
13889 %}
13890
13891 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13892 %{
13893 match(Set dst (RoundF src));
13894 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13895 format %{ "java_round_float $dst,$src"%}
13896 ins_encode %{
13897 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13898 as_FloatRegister($ftmp$$reg));
13899 %}
13900 ins_pipe(pipe_slow);
13901 %}
13902
13903 // stack <-> reg and reg <-> reg shuffles with no conversion
13904
13905 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13906
13907 match(Set dst (MoveF2I src));
13908
13909 effect(DEF dst, USE src);
13910
13911 ins_cost(4 * INSN_COST);
13912
13913 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13914
13915 ins_encode %{
13916 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13917 %}
13918
13919 ins_pipe(iload_reg_reg);
13920
13921 %}
13922
13923 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13924
13925 match(Set dst (MoveI2F src));
13926
13927 effect(DEF dst, USE src);
13928
13929 ins_cost(4 * INSN_COST);
13930
13931 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13932
13933 ins_encode %{
13934 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13935 %}
13936
13937 ins_pipe(pipe_class_memory);
13938
13939 %}
13940
13941 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13942
13943 match(Set dst (MoveD2L src));
13944
13945 effect(DEF dst, USE src);
13946
13947 ins_cost(4 * INSN_COST);
13948
13949 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13950
13951 ins_encode %{
13952 __ ldr($dst$$Register, Address(sp, $src$$disp));
13953 %}
13954
13955 ins_pipe(iload_reg_reg);
13956
13957 %}
13958
13959 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13960
13961 match(Set dst (MoveL2D src));
13962
13963 effect(DEF dst, USE src);
13964
13965 ins_cost(4 * INSN_COST);
13966
13967 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13968
13969 ins_encode %{
13970 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13971 %}
13972
13973 ins_pipe(pipe_class_memory);
13974
13975 %}
13976
13977 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13978
13979 match(Set dst (MoveF2I src));
13980
13981 effect(DEF dst, USE src);
13982
13983 ins_cost(INSN_COST);
13984
13985 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13986
13987 ins_encode %{
13988 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13989 %}
13990
13991 ins_pipe(pipe_class_memory);
13992
13993 %}
13994
13995 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13996
13997 match(Set dst (MoveI2F src));
13998
13999 effect(DEF dst, USE src);
14000
14001 ins_cost(INSN_COST);
14002
14003 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14004
14005 ins_encode %{
14006 __ strw($src$$Register, Address(sp, $dst$$disp));
14007 %}
14008
14009 ins_pipe(istore_reg_reg);
14010
14011 %}
14012
14013 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14014
14015 match(Set dst (MoveD2L src));
14016
14017 effect(DEF dst, USE src);
14018
14019 ins_cost(INSN_COST);
14020
14021 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14022
14023 ins_encode %{
14024 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14025 %}
14026
14027 ins_pipe(pipe_class_memory);
14028
14029 %}
14030
14031 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14032
14033 match(Set dst (MoveL2D src));
14034
14035 effect(DEF dst, USE src);
14036
14037 ins_cost(INSN_COST);
14038
14039 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14040
14041 ins_encode %{
14042 __ str($src$$Register, Address(sp, $dst$$disp));
14043 %}
14044
14045 ins_pipe(istore_reg_reg);
14046
14047 %}
14048
14049 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14050
14051 match(Set dst (MoveF2I src));
14052
14053 effect(DEF dst, USE src);
14054
14055 ins_cost(INSN_COST);
14056
14057 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14058
14059 ins_encode %{
14060 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14061 %}
14062
14063 ins_pipe(fp_f2i);
14064
14065 %}
14066
14067 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14068
14069 match(Set dst (MoveI2F src));
14070
14071 effect(DEF dst, USE src);
14072
14073 ins_cost(INSN_COST);
14074
14075 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14076
14077 ins_encode %{
14078 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14079 %}
14080
14081 ins_pipe(fp_i2f);
14082
14083 %}
14084
14085 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14086
14087 match(Set dst (MoveD2L src));
14088
14089 effect(DEF dst, USE src);
14090
14091 ins_cost(INSN_COST);
14092
14093 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14094
14095 ins_encode %{
14096 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14097 %}
14098
14099 ins_pipe(fp_d2l);
14100
14101 %}
14102
14103 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14104
14105 match(Set dst (MoveL2D src));
14106
14107 effect(DEF dst, USE src);
14108
14109 ins_cost(INSN_COST);
14110
14111 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14112
14113 ins_encode %{
14114 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14115 %}
14116
14117 ins_pipe(fp_l2d);
14118
14119 %}
14120
14121 // ============================================================================
14122 // clearing of an array
14123
14124 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14125 %{
14126 match(Set dummy (ClearArray (Binary cnt base) zero));
14127 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14128
14129 ins_cost(4 * INSN_COST);
14130 format %{ "ClearArray $cnt, $base" %}
14131
14132 ins_encode %{
14133 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14134 if (tpc == nullptr) {
14135 ciEnv::current()->record_failure("CodeCache is full");
14136 return;
14137 }
14138 %}
14139
14140 ins_pipe(pipe_class_memory);
14141 %}
14142
14143 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14144 %{
14145 predicate(((ClearArrayNode*)n)->word_copy_only());
14146 match(Set dummy (ClearArray (Binary cnt base) val));
14147 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14148
14149 ins_cost(4 * INSN_COST);
14150 format %{ "ClearArray $cnt, $base, $val" %}
14151
14152 ins_encode %{
14153 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14154 %}
14155
14156 ins_pipe(pipe_class_memory);
14157 %}
14158
14159 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14160 %{
14161 predicate((uint64_t)n->in(2)->get_long()
14162 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14163 && !((ClearArrayNode*)n)->word_copy_only());
14164 match(Set dummy (ClearArray cnt base));
14165 effect(TEMP temp, USE_KILL base, KILL cr);
14166
14167 ins_cost(4 * INSN_COST);
14168 format %{ "ClearArray $cnt, $base" %}
14169
14170 ins_encode %{
14171 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14172 if (tpc == nullptr) {
14173 ciEnv::current()->record_failure("CodeCache is full");
14174 return;
14175 }
14176 %}
14177
14178 ins_pipe(pipe_class_memory);
14179 %}
14180
14181 // ============================================================================
14182 // Overflow Math Instructions
14183
14184 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14185 %{
14186 match(Set cr (OverflowAddI op1 op2));
14187
14188 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14189 ins_cost(INSN_COST);
14190 ins_encode %{
14191 __ cmnw($op1$$Register, $op2$$Register);
14192 %}
14193
14194 ins_pipe(icmp_reg_reg);
14195 %}
14196
14197 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14198 %{
14199 match(Set cr (OverflowAddI op1 op2));
14200
14201 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14202 ins_cost(INSN_COST);
14203 ins_encode %{
14204 __ cmnw($op1$$Register, $op2$$constant);
14205 %}
14206
14207 ins_pipe(icmp_reg_imm);
14208 %}
14209
14210 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14211 %{
14212 match(Set cr (OverflowAddL op1 op2));
14213
14214 format %{ "cmn $op1, $op2\t# overflow check long" %}
14215 ins_cost(INSN_COST);
14216 ins_encode %{
14217 __ cmn($op1$$Register, $op2$$Register);
14218 %}
14219
14220 ins_pipe(icmp_reg_reg);
14221 %}
14222
14223 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14224 %{
14225 match(Set cr (OverflowAddL op1 op2));
14226
14227 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14228 ins_cost(INSN_COST);
14229 ins_encode %{
14230 __ adds(zr, $op1$$Register, $op2$$constant);
14231 %}
14232
14233 ins_pipe(icmp_reg_imm);
14234 %}
14235
14236 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14237 %{
14238 match(Set cr (OverflowSubI op1 op2));
14239
14240 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14241 ins_cost(INSN_COST);
14242 ins_encode %{
14243 __ cmpw($op1$$Register, $op2$$Register);
14244 %}
14245
14246 ins_pipe(icmp_reg_reg);
14247 %}
14248
14249 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14250 %{
14251 match(Set cr (OverflowSubI op1 op2));
14252
14253 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14254 ins_cost(INSN_COST);
14255 ins_encode %{
14256 __ cmpw($op1$$Register, $op2$$constant);
14257 %}
14258
14259 ins_pipe(icmp_reg_imm);
14260 %}
14261
14262 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14263 %{
14264 match(Set cr (OverflowSubL op1 op2));
14265
14266 format %{ "cmp $op1, $op2\t# overflow check long" %}
14267 ins_cost(INSN_COST);
14268 ins_encode %{
14269 __ cmp($op1$$Register, $op2$$Register);
14270 %}
14271
14272 ins_pipe(icmp_reg_reg);
14273 %}
14274
14275 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14276 %{
14277 match(Set cr (OverflowSubL op1 op2));
14278
14279 format %{ "cmp $op1, $op2\t# overflow check long" %}
14280 ins_cost(INSN_COST);
14281 ins_encode %{
14282 __ subs(zr, $op1$$Register, $op2$$constant);
14283 %}
14284
14285 ins_pipe(icmp_reg_imm);
14286 %}
14287
14288 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14289 %{
14290 match(Set cr (OverflowSubI zero op1));
14291
14292 format %{ "cmpw zr, $op1\t# overflow check int" %}
14293 ins_cost(INSN_COST);
14294 ins_encode %{
14295 __ cmpw(zr, $op1$$Register);
14296 %}
14297
14298 ins_pipe(icmp_reg_imm);
14299 %}
14300
14301 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14302 %{
14303 match(Set cr (OverflowSubL zero op1));
14304
14305 format %{ "cmp zr, $op1\t# overflow check long" %}
14306 ins_cost(INSN_COST);
14307 ins_encode %{
14308 __ cmp(zr, $op1$$Register);
14309 %}
14310
14311 ins_pipe(icmp_reg_imm);
14312 %}
14313
14314 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14315 %{
14316 match(Set cr (OverflowMulI op1 op2));
14317
14318 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14319 "cmp rscratch1, rscratch1, sxtw\n\t"
14320 "movw rscratch1, #0x80000000\n\t"
14321 "cselw rscratch1, rscratch1, zr, NE\n\t"
14322 "cmpw rscratch1, #1" %}
14323 ins_cost(5 * INSN_COST);
14324 ins_encode %{
14325 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14326 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14327 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14328 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14329 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14330 %}
14331
14332 ins_pipe(pipe_slow);
14333 %}
14334
14335 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14336 %{
14337 match(If cmp (OverflowMulI op1 op2));
14338 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14339 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14340 effect(USE labl, KILL cr);
14341
14342 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14343 "cmp rscratch1, rscratch1, sxtw\n\t"
14344 "b$cmp $labl" %}
14345 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14346 ins_encode %{
14347 Label* L = $labl$$label;
14348 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14349 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14350 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14351 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14352 %}
14353
14354 ins_pipe(pipe_serial);
14355 %}
14356
14357 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14358 %{
14359 match(Set cr (OverflowMulL op1 op2));
14360
14361 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14362 "smulh rscratch2, $op1, $op2\n\t"
14363 "cmp rscratch2, rscratch1, ASR #63\n\t"
14364 "movw rscratch1, #0x80000000\n\t"
14365 "cselw rscratch1, rscratch1, zr, NE\n\t"
14366 "cmpw rscratch1, #1" %}
14367 ins_cost(6 * INSN_COST);
14368 ins_encode %{
14369 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14370 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14371 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14372 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14373 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14374 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14375 %}
14376
14377 ins_pipe(pipe_slow);
14378 %}
14379
14380 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14381 %{
14382 match(If cmp (OverflowMulL op1 op2));
14383 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14384 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14385 effect(USE labl, KILL cr);
14386
14387 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14388 "smulh rscratch2, $op1, $op2\n\t"
14389 "cmp rscratch2, rscratch1, ASR #63\n\t"
14390 "b$cmp $labl" %}
14391 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14392 ins_encode %{
14393 Label* L = $labl$$label;
14394 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14395 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14396 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14397 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14398 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14399 %}
14400
14401 ins_pipe(pipe_serial);
14402 %}
14403
14404 // ============================================================================
14405 // Compare Instructions
14406
14407 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14408 %{
14409 match(Set cr (CmpI op1 op2));
14410
14411 effect(DEF cr, USE op1, USE op2);
14412
14413 ins_cost(INSN_COST);
14414 format %{ "cmpw $op1, $op2" %}
14415
14416 ins_encode(aarch64_enc_cmpw(op1, op2));
14417
14418 ins_pipe(icmp_reg_reg);
14419 %}
14420
14421 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14422 %{
14423 match(Set cr (CmpI op1 zero));
14424
14425 effect(DEF cr, USE op1);
14426
14427 ins_cost(INSN_COST);
14428 format %{ "cmpw $op1, 0" %}
14429
14430 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14431
14432 ins_pipe(icmp_reg_imm);
14433 %}
14434
14435 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14436 %{
14437 match(Set cr (CmpI op1 op2));
14438
14439 effect(DEF cr, USE op1);
14440
14441 ins_cost(INSN_COST);
14442 format %{ "cmpw $op1, $op2" %}
14443
14444 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14445
14446 ins_pipe(icmp_reg_imm);
14447 %}
14448
14449 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14450 %{
14451 match(Set cr (CmpI op1 op2));
14452
14453 effect(DEF cr, USE op1);
14454
14455 ins_cost(INSN_COST * 2);
14456 format %{ "cmpw $op1, $op2" %}
14457
14458 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14459
14460 ins_pipe(icmp_reg_imm);
14461 %}
14462
14463 // Unsigned compare Instructions; really, same as signed compare
14464 // except it should only be used to feed an If or a CMovI which takes a
14465 // cmpOpU.
14466
14467 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14468 %{
14469 match(Set cr (CmpU op1 op2));
14470
14471 effect(DEF cr, USE op1, USE op2);
14472
14473 ins_cost(INSN_COST);
14474 format %{ "cmpw $op1, $op2\t# unsigned" %}
14475
14476 ins_encode(aarch64_enc_cmpw(op1, op2));
14477
14478 ins_pipe(icmp_reg_reg);
14479 %}
14480
14481 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14482 %{
14483 match(Set cr (CmpU op1 zero));
14484
14485 effect(DEF cr, USE op1);
14486
14487 ins_cost(INSN_COST);
14488 format %{ "cmpw $op1, #0\t# unsigned" %}
14489
14490 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14491
14492 ins_pipe(icmp_reg_imm);
14493 %}
14494
14495 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14496 %{
14497 match(Set cr (CmpU op1 op2));
14498
14499 effect(DEF cr, USE op1);
14500
14501 ins_cost(INSN_COST);
14502 format %{ "cmpw $op1, $op2\t# unsigned" %}
14503
14504 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14505
14506 ins_pipe(icmp_reg_imm);
14507 %}
14508
14509 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14510 %{
14511 match(Set cr (CmpU op1 op2));
14512
14513 effect(DEF cr, USE op1);
14514
14515 ins_cost(INSN_COST * 2);
14516 format %{ "cmpw $op1, $op2\t# unsigned" %}
14517
14518 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14519
14520 ins_pipe(icmp_reg_imm);
14521 %}
14522
14523 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14524 %{
14525 match(Set cr (CmpL op1 op2));
14526
14527 effect(DEF cr, USE op1, USE op2);
14528
14529 ins_cost(INSN_COST);
14530 format %{ "cmp $op1, $op2" %}
14531
14532 ins_encode(aarch64_enc_cmp(op1, op2));
14533
14534 ins_pipe(icmp_reg_reg);
14535 %}
14536
14537 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14538 %{
14539 match(Set cr (CmpL op1 zero));
14540
14541 effect(DEF cr, USE op1);
14542
14543 ins_cost(INSN_COST);
14544 format %{ "tst $op1" %}
14545
14546 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14547
14548 ins_pipe(icmp_reg_imm);
14549 %}
14550
14551 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14552 %{
14553 match(Set cr (CmpL op1 op2));
14554
14555 effect(DEF cr, USE op1);
14556
14557 ins_cost(INSN_COST);
14558 format %{ "cmp $op1, $op2" %}
14559
14560 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14561
14562 ins_pipe(icmp_reg_imm);
14563 %}
14564
14565 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14566 %{
14567 match(Set cr (CmpL op1 op2));
14568
14569 effect(DEF cr, USE op1);
14570
14571 ins_cost(INSN_COST * 2);
14572 format %{ "cmp $op1, $op2" %}
14573
14574 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14575
14576 ins_pipe(icmp_reg_imm);
14577 %}
14578
14579 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14580 %{
14581 match(Set cr (CmpUL op1 op2));
14582
14583 effect(DEF cr, USE op1, USE op2);
14584
14585 ins_cost(INSN_COST);
14586 format %{ "cmp $op1, $op2" %}
14587
14588 ins_encode(aarch64_enc_cmp(op1, op2));
14589
14590 ins_pipe(icmp_reg_reg);
14591 %}
14592
14593 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14594 %{
14595 match(Set cr (CmpUL op1 zero));
14596
14597 effect(DEF cr, USE op1);
14598
14599 ins_cost(INSN_COST);
14600 format %{ "tst $op1" %}
14601
14602 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14603
14604 ins_pipe(icmp_reg_imm);
14605 %}
14606
14607 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14608 %{
14609 match(Set cr (CmpUL op1 op2));
14610
14611 effect(DEF cr, USE op1);
14612
14613 ins_cost(INSN_COST);
14614 format %{ "cmp $op1, $op2" %}
14615
14616 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14617
14618 ins_pipe(icmp_reg_imm);
14619 %}
14620
14621 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14622 %{
14623 match(Set cr (CmpUL op1 op2));
14624
14625 effect(DEF cr, USE op1);
14626
14627 ins_cost(INSN_COST * 2);
14628 format %{ "cmp $op1, $op2" %}
14629
14630 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14631
14632 ins_pipe(icmp_reg_imm);
14633 %}
14634
14635 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14636 %{
14637 match(Set cr (CmpP op1 op2));
14638
14639 effect(DEF cr, USE op1, USE op2);
14640
14641 ins_cost(INSN_COST);
14642 format %{ "cmp $op1, $op2\t // ptr" %}
14643
14644 ins_encode(aarch64_enc_cmpp(op1, op2));
14645
14646 ins_pipe(icmp_reg_reg);
14647 %}
14648
14649 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14650 %{
14651 match(Set cr (CmpN op1 op2));
14652
14653 effect(DEF cr, USE op1, USE op2);
14654
14655 ins_cost(INSN_COST);
14656 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14657
14658 ins_encode(aarch64_enc_cmpn(op1, op2));
14659
14660 ins_pipe(icmp_reg_reg);
14661 %}
14662
14663 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14664 %{
14665 match(Set cr (CmpP op1 zero));
14666
14667 effect(DEF cr, USE op1, USE zero);
14668
14669 ins_cost(INSN_COST);
14670 format %{ "cmp $op1, 0\t // ptr" %}
14671
14672 ins_encode(aarch64_enc_testp(op1));
14673
14674 ins_pipe(icmp_reg_imm);
14675 %}
14676
14677 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14678 %{
14679 match(Set cr (CmpN op1 zero));
14680
14681 effect(DEF cr, USE op1, USE zero);
14682
14683 ins_cost(INSN_COST);
14684 format %{ "cmp $op1, 0\t // compressed ptr" %}
14685
14686 ins_encode(aarch64_enc_testn(op1));
14687
14688 ins_pipe(icmp_reg_imm);
14689 %}
14690
14691 // FP comparisons
14692 //
14693 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14694 // using normal cmpOp. See declaration of rFlagsReg for details.
14695
14696 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14697 %{
14698 match(Set cr (CmpF src1 src2));
14699
14700 ins_cost(3 * INSN_COST);
14701 format %{ "fcmps $src1, $src2" %}
14702
14703 ins_encode %{
14704 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14705 %}
14706
14707 ins_pipe(pipe_class_compare);
14708 %}
14709
14710 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14711 %{
14712 match(Set cr (CmpF src1 src2));
14713
14714 ins_cost(3 * INSN_COST);
14715 format %{ "fcmps $src1, 0.0" %}
14716
14717 ins_encode %{
14718 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14719 %}
14720
14721 ins_pipe(pipe_class_compare);
14722 %}
14723 // FROM HERE
14724
14725 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14726 %{
14727 match(Set cr (CmpD src1 src2));
14728
14729 ins_cost(3 * INSN_COST);
14730 format %{ "fcmpd $src1, $src2" %}
14731
14732 ins_encode %{
14733 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14734 %}
14735
14736 ins_pipe(pipe_class_compare);
14737 %}
14738
14739 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14740 %{
14741 match(Set cr (CmpD src1 src2));
14742
14743 ins_cost(3 * INSN_COST);
14744 format %{ "fcmpd $src1, 0.0" %}
14745
14746 ins_encode %{
14747 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14748 %}
14749
14750 ins_pipe(pipe_class_compare);
14751 %}
14752
14753 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14754 %{
14755 match(Set dst (CmpF3 src1 src2));
14756 effect(KILL cr);
14757
14758 ins_cost(5 * INSN_COST);
14759 format %{ "fcmps $src1, $src2\n\t"
14760 "csinvw($dst, zr, zr, eq\n\t"
14761 "csnegw($dst, $dst, $dst, lt)"
14762 %}
14763
14764 ins_encode %{
14765 Label done;
14766 FloatRegister s1 = as_FloatRegister($src1$$reg);
14767 FloatRegister s2 = as_FloatRegister($src2$$reg);
14768 Register d = as_Register($dst$$reg);
14769 __ fcmps(s1, s2);
14770 // installs 0 if EQ else -1
14771 __ csinvw(d, zr, zr, Assembler::EQ);
14772 // keeps -1 if less or unordered else installs 1
14773 __ csnegw(d, d, d, Assembler::LT);
14774 __ bind(done);
14775 %}
14776
14777 ins_pipe(pipe_class_default);
14778
14779 %}
14780
14781 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14782 %{
14783 match(Set dst (CmpD3 src1 src2));
14784 effect(KILL cr);
14785
14786 ins_cost(5 * INSN_COST);
14787 format %{ "fcmpd $src1, $src2\n\t"
14788 "csinvw($dst, zr, zr, eq\n\t"
14789 "csnegw($dst, $dst, $dst, lt)"
14790 %}
14791
14792 ins_encode %{
14793 Label done;
14794 FloatRegister s1 = as_FloatRegister($src1$$reg);
14795 FloatRegister s2 = as_FloatRegister($src2$$reg);
14796 Register d = as_Register($dst$$reg);
14797 __ fcmpd(s1, s2);
14798 // installs 0 if EQ else -1
14799 __ csinvw(d, zr, zr, Assembler::EQ);
14800 // keeps -1 if less or unordered else installs 1
14801 __ csnegw(d, d, d, Assembler::LT);
14802 __ bind(done);
14803 %}
14804 ins_pipe(pipe_class_default);
14805
14806 %}
14807
14808 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14809 %{
14810 match(Set dst (CmpF3 src1 zero));
14811 effect(KILL cr);
14812
14813 ins_cost(5 * INSN_COST);
14814 format %{ "fcmps $src1, 0.0\n\t"
14815 "csinvw($dst, zr, zr, eq\n\t"
14816 "csnegw($dst, $dst, $dst, lt)"
14817 %}
14818
14819 ins_encode %{
14820 Label done;
14821 FloatRegister s1 = as_FloatRegister($src1$$reg);
14822 Register d = as_Register($dst$$reg);
14823 __ fcmps(s1, 0.0);
14824 // installs 0 if EQ else -1
14825 __ csinvw(d, zr, zr, Assembler::EQ);
14826 // keeps -1 if less or unordered else installs 1
14827 __ csnegw(d, d, d, Assembler::LT);
14828 __ bind(done);
14829 %}
14830
14831 ins_pipe(pipe_class_default);
14832
14833 %}
14834
14835 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14836 %{
14837 match(Set dst (CmpD3 src1 zero));
14838 effect(KILL cr);
14839
14840 ins_cost(5 * INSN_COST);
14841 format %{ "fcmpd $src1, 0.0\n\t"
14842 "csinvw($dst, zr, zr, eq\n\t"
14843 "csnegw($dst, $dst, $dst, lt)"
14844 %}
14845
14846 ins_encode %{
14847 Label done;
14848 FloatRegister s1 = as_FloatRegister($src1$$reg);
14849 Register d = as_Register($dst$$reg);
14850 __ fcmpd(s1, 0.0);
14851 // installs 0 if EQ else -1
14852 __ csinvw(d, zr, zr, Assembler::EQ);
14853 // keeps -1 if less or unordered else installs 1
14854 __ csnegw(d, d, d, Assembler::LT);
14855 __ bind(done);
14856 %}
14857 ins_pipe(pipe_class_default);
14858
14859 %}
14860
14861 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14862 %{
14863 match(Set dst (CmpLTMask p q));
14864 effect(KILL cr);
14865
14866 ins_cost(3 * INSN_COST);
14867
14868 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14869 "csetw $dst, lt\n\t"
14870 "subw $dst, zr, $dst"
14871 %}
14872
14873 ins_encode %{
14874 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14875 __ csetw(as_Register($dst$$reg), Assembler::LT);
14876 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14877 %}
14878
14879 ins_pipe(ialu_reg_reg);
14880 %}
14881
14882 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14883 %{
14884 match(Set dst (CmpLTMask src zero));
14885 effect(KILL cr);
14886
14887 ins_cost(INSN_COST);
14888
14889 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14890
14891 ins_encode %{
14892 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14893 %}
14894
14895 ins_pipe(ialu_reg_shift);
14896 %}
14897
14898 // ============================================================================
14899 // Max and Min
14900
14901 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14902
14903 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14904 %{
14905 effect(DEF cr, USE src);
14906 ins_cost(INSN_COST);
14907 format %{ "cmpw $src, 0" %}
14908
14909 ins_encode %{
14910 __ cmpw($src$$Register, 0);
14911 %}
14912 ins_pipe(icmp_reg_imm);
14913 %}
14914
14915 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14916 %{
14917 match(Set dst (MinI src1 src2));
14918 ins_cost(INSN_COST * 3);
14919
14920 expand %{
14921 rFlagsReg cr;
14922 compI_reg_reg(cr, src1, src2);
14923 cmovI_reg_reg_lt(dst, src1, src2, cr);
14924 %}
14925 %}
14926
14927 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14928 %{
14929 match(Set dst (MaxI src1 src2));
14930 ins_cost(INSN_COST * 3);
14931
14932 expand %{
14933 rFlagsReg cr;
14934 compI_reg_reg(cr, src1, src2);
14935 cmovI_reg_reg_gt(dst, src1, src2, cr);
14936 %}
14937 %}
14938
14939
14940 // ============================================================================
14941 // Branch Instructions
14942
14943 // Direct Branch.
14944 instruct branch(label lbl)
14945 %{
14946 match(Goto);
14947
14948 effect(USE lbl);
14949
14950 ins_cost(BRANCH_COST);
14951 format %{ "b $lbl" %}
14952
14953 ins_encode(aarch64_enc_b(lbl));
14954
14955 ins_pipe(pipe_branch);
14956 %}
14957
14958 // Conditional Near Branch
14959 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14960 %{
14961 // Same match rule as `branchConFar'.
14962 match(If cmp cr);
14963
14964 effect(USE lbl);
14965
14966 ins_cost(BRANCH_COST);
14967 // If set to 1 this indicates that the current instruction is a
14968 // short variant of a long branch. This avoids using this
14969 // instruction in first-pass matching. It will then only be used in
14970 // the `Shorten_branches' pass.
14971 // ins_short_branch(1);
14972 format %{ "b$cmp $lbl" %}
14973
14974 ins_encode(aarch64_enc_br_con(cmp, lbl));
14975
14976 ins_pipe(pipe_branch_cond);
14977 %}
14978
14979 // Conditional Near Branch Unsigned
14980 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14981 %{
14982 // Same match rule as `branchConFar'.
14983 match(If cmp cr);
14984
14985 effect(USE lbl);
14986
14987 ins_cost(BRANCH_COST);
14988 // If set to 1 this indicates that the current instruction is a
14989 // short variant of a long branch. This avoids using this
14990 // instruction in first-pass matching. It will then only be used in
14991 // the `Shorten_branches' pass.
14992 // ins_short_branch(1);
14993 format %{ "b$cmp $lbl\t# unsigned" %}
14994
14995 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14996
14997 ins_pipe(pipe_branch_cond);
14998 %}
14999
15000 // Make use of CBZ and CBNZ. These instructions, as well as being
15001 // shorter than (cmp; branch), have the additional benefit of not
15002 // killing the flags.
15003
15004 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15005 match(If cmp (CmpI op1 op2));
15006 effect(USE labl);
15007
15008 ins_cost(BRANCH_COST);
15009 format %{ "cbw$cmp $op1, $labl" %}
15010 ins_encode %{
15011 Label* L = $labl$$label;
15012 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15013 if (cond == Assembler::EQ)
15014 __ cbzw($op1$$Register, *L);
15015 else
15016 __ cbnzw($op1$$Register, *L);
15017 %}
15018 ins_pipe(pipe_cmp_branch);
15019 %}
15020
15021 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15022 match(If cmp (CmpL op1 op2));
15023 effect(USE labl);
15024
15025 ins_cost(BRANCH_COST);
15026 format %{ "cb$cmp $op1, $labl" %}
15027 ins_encode %{
15028 Label* L = $labl$$label;
15029 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15030 if (cond == Assembler::EQ)
15031 __ cbz($op1$$Register, *L);
15032 else
15033 __ cbnz($op1$$Register, *L);
15034 %}
15035 ins_pipe(pipe_cmp_branch);
15036 %}
15037
15038 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15039 match(If cmp (CmpP op1 op2));
15040 effect(USE labl);
15041
15042 ins_cost(BRANCH_COST);
15043 format %{ "cb$cmp $op1, $labl" %}
15044 ins_encode %{
15045 Label* L = $labl$$label;
15046 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15047 if (cond == Assembler::EQ)
15048 __ cbz($op1$$Register, *L);
15049 else
15050 __ cbnz($op1$$Register, *L);
15051 %}
15052 ins_pipe(pipe_cmp_branch);
15053 %}
15054
15055 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15056 match(If cmp (CmpN op1 op2));
15057 effect(USE labl);
15058
15059 ins_cost(BRANCH_COST);
15060 format %{ "cbw$cmp $op1, $labl" %}
15061 ins_encode %{
15062 Label* L = $labl$$label;
15063 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15064 if (cond == Assembler::EQ)
15065 __ cbzw($op1$$Register, *L);
15066 else
15067 __ cbnzw($op1$$Register, *L);
15068 %}
15069 ins_pipe(pipe_cmp_branch);
15070 %}
15071
15072 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15073 match(If cmp (CmpP (DecodeN oop) zero));
15074 effect(USE labl);
15075
15076 ins_cost(BRANCH_COST);
15077 format %{ "cb$cmp $oop, $labl" %}
15078 ins_encode %{
15079 Label* L = $labl$$label;
15080 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15081 if (cond == Assembler::EQ)
15082 __ cbzw($oop$$Register, *L);
15083 else
15084 __ cbnzw($oop$$Register, *L);
15085 %}
15086 ins_pipe(pipe_cmp_branch);
15087 %}
15088
15089 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15090 match(If cmp (CmpU op1 op2));
15091 effect(USE labl);
15092
15093 ins_cost(BRANCH_COST);
15094 format %{ "cbw$cmp $op1, $labl" %}
15095 ins_encode %{
15096 Label* L = $labl$$label;
15097 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15098 if (cond == Assembler::EQ || cond == Assembler::LS) {
15099 __ cbzw($op1$$Register, *L);
15100 } else {
15101 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15102 __ cbnzw($op1$$Register, *L);
15103 }
15104 %}
15105 ins_pipe(pipe_cmp_branch);
15106 %}
15107
15108 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15109 match(If cmp (CmpUL op1 op2));
15110 effect(USE labl);
15111
15112 ins_cost(BRANCH_COST);
15113 format %{ "cb$cmp $op1, $labl" %}
15114 ins_encode %{
15115 Label* L = $labl$$label;
15116 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15117 if (cond == Assembler::EQ || cond == Assembler::LS) {
15118 __ cbz($op1$$Register, *L);
15119 } else {
15120 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15121 __ cbnz($op1$$Register, *L);
15122 }
15123 %}
15124 ins_pipe(pipe_cmp_branch);
15125 %}
15126
15127 // Test bit and Branch
15128
15129 // Patterns for short (< 32KiB) variants
15130 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15131 match(If cmp (CmpL op1 op2));
15132 effect(USE labl);
15133
15134 ins_cost(BRANCH_COST);
15135 format %{ "cb$cmp $op1, $labl # long" %}
15136 ins_encode %{
15137 Label* L = $labl$$label;
15138 Assembler::Condition cond =
15139 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15140 __ tbr(cond, $op1$$Register, 63, *L);
15141 %}
15142 ins_pipe(pipe_cmp_branch);
15143 ins_short_branch(1);
15144 %}
15145
15146 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15147 match(If cmp (CmpI op1 op2));
15148 effect(USE labl);
15149
15150 ins_cost(BRANCH_COST);
15151 format %{ "cb$cmp $op1, $labl # int" %}
15152 ins_encode %{
15153 Label* L = $labl$$label;
15154 Assembler::Condition cond =
15155 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15156 __ tbr(cond, $op1$$Register, 31, *L);
15157 %}
15158 ins_pipe(pipe_cmp_branch);
15159 ins_short_branch(1);
15160 %}
15161
15162 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15163 match(If cmp (CmpL (AndL op1 op2) op3));
15164 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15165 effect(USE labl);
15166
15167 ins_cost(BRANCH_COST);
15168 format %{ "tb$cmp $op1, $op2, $labl" %}
15169 ins_encode %{
15170 Label* L = $labl$$label;
15171 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15172 int bit = exact_log2_long($op2$$constant);
15173 __ tbr(cond, $op1$$Register, bit, *L);
15174 %}
15175 ins_pipe(pipe_cmp_branch);
15176 ins_short_branch(1);
15177 %}
15178
15179 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15180 match(If cmp (CmpI (AndI op1 op2) op3));
15181 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15182 effect(USE labl);
15183
15184 ins_cost(BRANCH_COST);
15185 format %{ "tb$cmp $op1, $op2, $labl" %}
15186 ins_encode %{
15187 Label* L = $labl$$label;
15188 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15189 int bit = exact_log2((juint)$op2$$constant);
15190 __ tbr(cond, $op1$$Register, bit, *L);
15191 %}
15192 ins_pipe(pipe_cmp_branch);
15193 ins_short_branch(1);
15194 %}
15195
15196 // And far variants
15197 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15198 match(If cmp (CmpL op1 op2));
15199 effect(USE labl);
15200
15201 ins_cost(BRANCH_COST);
15202 format %{ "cb$cmp $op1, $labl # long" %}
15203 ins_encode %{
15204 Label* L = $labl$$label;
15205 Assembler::Condition cond =
15206 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15207 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15208 %}
15209 ins_pipe(pipe_cmp_branch);
15210 %}
15211
15212 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15213 match(If cmp (CmpI op1 op2));
15214 effect(USE labl);
15215
15216 ins_cost(BRANCH_COST);
15217 format %{ "cb$cmp $op1, $labl # int" %}
15218 ins_encode %{
15219 Label* L = $labl$$label;
15220 Assembler::Condition cond =
15221 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15222 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15223 %}
15224 ins_pipe(pipe_cmp_branch);
15225 %}
15226
15227 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15228 match(If cmp (CmpL (AndL op1 op2) op3));
15229 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15230 effect(USE labl);
15231
15232 ins_cost(BRANCH_COST);
15233 format %{ "tb$cmp $op1, $op2, $labl" %}
15234 ins_encode %{
15235 Label* L = $labl$$label;
15236 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15237 int bit = exact_log2_long($op2$$constant);
15238 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15239 %}
15240 ins_pipe(pipe_cmp_branch);
15241 %}
15242
15243 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15244 match(If cmp (CmpI (AndI op1 op2) op3));
15245 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15246 effect(USE labl);
15247
15248 ins_cost(BRANCH_COST);
15249 format %{ "tb$cmp $op1, $op2, $labl" %}
15250 ins_encode %{
15251 Label* L = $labl$$label;
15252 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15253 int bit = exact_log2((juint)$op2$$constant);
15254 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15255 %}
15256 ins_pipe(pipe_cmp_branch);
15257 %}
15258
15259 // Test bits
15260
15261 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15262 match(Set cr (CmpL (AndL op1 op2) op3));
15263 predicate(Assembler::operand_valid_for_logical_immediate
15264 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15265
15266 ins_cost(INSN_COST);
15267 format %{ "tst $op1, $op2 # long" %}
15268 ins_encode %{
15269 __ tst($op1$$Register, $op2$$constant);
15270 %}
15271 ins_pipe(ialu_reg_reg);
15272 %}
15273
15274 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15275 match(Set cr (CmpI (AndI op1 op2) op3));
15276 predicate(Assembler::operand_valid_for_logical_immediate
15277 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15278
15279 ins_cost(INSN_COST);
15280 format %{ "tst $op1, $op2 # int" %}
15281 ins_encode %{
15282 __ tstw($op1$$Register, $op2$$constant);
15283 %}
15284 ins_pipe(ialu_reg_reg);
15285 %}
15286
15287 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15288 match(Set cr (CmpL (AndL op1 op2) op3));
15289
15290 ins_cost(INSN_COST);
15291 format %{ "tst $op1, $op2 # long" %}
15292 ins_encode %{
15293 __ tst($op1$$Register, $op2$$Register);
15294 %}
15295 ins_pipe(ialu_reg_reg);
15296 %}
15297
15298 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15299 match(Set cr (CmpI (AndI op1 op2) op3));
15300
15301 ins_cost(INSN_COST);
15302 format %{ "tstw $op1, $op2 # int" %}
15303 ins_encode %{
15304 __ tstw($op1$$Register, $op2$$Register);
15305 %}
15306 ins_pipe(ialu_reg_reg);
15307 %}
15308
15309
15310 // Conditional Far Branch
15311 // Conditional Far Branch Unsigned
15312 // TODO: fixme
15313
15314 // counted loop end branch near
15315 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15316 %{
15317 match(CountedLoopEnd cmp cr);
15318
15319 effect(USE lbl);
15320
15321 ins_cost(BRANCH_COST);
15322 // short variant.
15323 // ins_short_branch(1);
15324 format %{ "b$cmp $lbl \t// counted loop end" %}
15325
15326 ins_encode(aarch64_enc_br_con(cmp, lbl));
15327
15328 ins_pipe(pipe_branch);
15329 %}
15330
15331 // counted loop end branch far
15332 // TODO: fixme
15333
15334 // ============================================================================
15335 // inlined locking and unlocking
15336
15337 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15338 %{
15339 match(Set cr (FastLock object box));
15340 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15341
15342 ins_cost(5 * INSN_COST);
15343 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15344
15345 ins_encode %{
15346 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15347 %}
15348
15349 ins_pipe(pipe_serial);
15350 %}
15351
15352 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15353 %{
15354 match(Set cr (FastUnlock object box));
15355 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15356
15357 ins_cost(5 * INSN_COST);
15358 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15359
15360 ins_encode %{
15361 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15362 %}
15363
15364 ins_pipe(pipe_serial);
15365 %}
15366
15367 // ============================================================================
15368 // Safepoint Instructions
15369
15370 // TODO
15371 // provide a near and far version of this code
15372
15373 instruct safePoint(rFlagsReg cr, iRegP poll)
15374 %{
15375 match(SafePoint poll);
15376 effect(KILL cr);
15377
15378 format %{
15379 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15380 %}
15381 ins_encode %{
15382 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15383 %}
15384 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15385 %}
15386
15387
15388 // ============================================================================
15389 // Procedure Call/Return Instructions
15390
15391 // Call Java Static Instruction
15392
15393 instruct CallStaticJavaDirect(method meth)
15394 %{
15395 match(CallStaticJava);
15396
15397 effect(USE meth);
15398
15399 ins_cost(CALL_COST);
15400
15401 format %{ "call,static $meth \t// ==> " %}
15402
15403 ins_encode(aarch64_enc_java_static_call(meth),
15404 aarch64_enc_call_epilog);
15405
15406 ins_pipe(pipe_class_call);
15407 %}
15408
15409 // TO HERE
15410
15411 // Call Java Dynamic Instruction
15412 instruct CallDynamicJavaDirect(method meth)
15413 %{
15414 match(CallDynamicJava);
15415
15416 effect(USE meth);
15417
15418 ins_cost(CALL_COST);
15419
15420 format %{ "CALL,dynamic $meth \t// ==> " %}
15421
15422 ins_encode(aarch64_enc_java_dynamic_call(meth),
15423 aarch64_enc_call_epilog);
15424
15425 ins_pipe(pipe_class_call);
15426 %}
15427
15428 // Call Runtime Instruction
15429
15430 instruct CallRuntimeDirect(method meth)
15431 %{
15432 match(CallRuntime);
15433
15434 effect(USE meth);
15435
15436 ins_cost(CALL_COST);
15437
15438 format %{ "CALL, runtime $meth" %}
15439
15440 ins_encode( aarch64_enc_java_to_runtime(meth) );
15441
15442 ins_pipe(pipe_class_call);
15443 %}
15444
15445 // Call Runtime Instruction
15446
15447 instruct CallLeafDirect(method meth)
15448 %{
15449 match(CallLeaf);
15450
15451 effect(USE meth);
15452
15453 ins_cost(CALL_COST);
15454
15455 format %{ "CALL, runtime leaf $meth" %}
15456
15457 ins_encode( aarch64_enc_java_to_runtime(meth) );
15458
15459 ins_pipe(pipe_class_call);
15460 %}
15461
15462 // Call Runtime Instruction without safepoint and with vector arguments
15463 instruct CallLeafDirectVector(method meth)
15464 %{
15465 match(CallLeafVector);
15466
15467 effect(USE meth);
15468
15469 ins_cost(CALL_COST);
15470
15471 format %{ "CALL, runtime leaf vector $meth" %}
15472
15473 ins_encode(aarch64_enc_java_to_runtime(meth));
15474
15475 ins_pipe(pipe_class_call);
15476 %}
15477
15478 // Call Runtime Instruction
15479
15480 // entry point is null, target holds the address to call
15481 instruct CallLeafNoFPIndirect(iRegP target)
15482 %{
15483 predicate(n->as_Call()->entry_point() == nullptr);
15484
15485 match(CallLeafNoFP target);
15486
15487 ins_cost(CALL_COST);
15488
15489 format %{ "CALL, runtime leaf nofp indirect $target" %}
15490
15491 ins_encode %{
15492 __ blr($target$$Register);
15493 %}
15494
15495 ins_pipe(pipe_class_call);
15496 %}
15497
15498 instruct CallLeafNoFPDirect(method meth)
15499 %{
15500 predicate(n->as_Call()->entry_point() != nullptr);
15501
15502 match(CallLeafNoFP);
15503
15504 effect(USE meth);
15505
15506 ins_cost(CALL_COST);
15507
15508 format %{ "CALL, runtime leaf nofp $meth" %}
15509
15510 ins_encode( aarch64_enc_java_to_runtime(meth) );
15511
15512 ins_pipe(pipe_class_call);
15513 %}
15514
15515 // Tail Call; Jump from runtime stub to Java code.
15516 // Also known as an 'interprocedural jump'.
15517 // Target of jump will eventually return to caller.
15518 // TailJump below removes the return address.
15519 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15520 // emitted just above the TailCall which has reset rfp to the caller state.
15521 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15522 %{
15523 match(TailCall jump_target method_ptr);
15524
15525 ins_cost(CALL_COST);
15526
15527 format %{ "br $jump_target\t# $method_ptr holds method" %}
15528
15529 ins_encode(aarch64_enc_tail_call(jump_target));
15530
15531 ins_pipe(pipe_class_call);
15532 %}
15533
15534 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15535 %{
15536 match(TailJump jump_target ex_oop);
15537
15538 ins_cost(CALL_COST);
15539
15540 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15541
15542 ins_encode(aarch64_enc_tail_jmp(jump_target));
15543
15544 ins_pipe(pipe_class_call);
15545 %}
15546
15547 // Forward exception.
15548 instruct ForwardExceptionjmp()
15549 %{
15550 match(ForwardException);
15551 ins_cost(CALL_COST);
15552
15553 format %{ "b forward_exception_stub" %}
15554 ins_encode %{
15555 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15556 %}
15557 ins_pipe(pipe_class_call);
15558 %}
15559
15560 // Create exception oop: created by stack-crawling runtime code.
15561 // Created exception is now available to this handler, and is setup
15562 // just prior to jumping to this handler. No code emitted.
15563 // TODO check
15564 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15565 instruct CreateException(iRegP_R0 ex_oop)
15566 %{
15567 match(Set ex_oop (CreateEx));
15568
15569 format %{ " -- \t// exception oop; no code emitted" %}
15570
15571 size(0);
15572
15573 ins_encode( /*empty*/ );
15574
15575 ins_pipe(pipe_class_empty);
15576 %}
15577
15578 // Rethrow exception: The exception oop will come in the first
15579 // argument position. Then JUMP (not call) to the rethrow stub code.
15580 instruct RethrowException() %{
15581 match(Rethrow);
15582 ins_cost(CALL_COST);
15583
15584 format %{ "b rethrow_stub" %}
15585
15586 ins_encode( aarch64_enc_rethrow() );
15587
15588 ins_pipe(pipe_class_call);
15589 %}
15590
15591
15592 // Return Instruction
15593 // epilog node loads ret address into lr as part of frame pop
15594 instruct Ret()
15595 %{
15596 match(Return);
15597
15598 format %{ "ret\t// return register" %}
15599
15600 ins_encode( aarch64_enc_ret() );
15601
15602 ins_pipe(pipe_branch);
15603 %}
15604
15605 // Die now.
15606 instruct ShouldNotReachHere() %{
15607 match(Halt);
15608
15609 ins_cost(CALL_COST);
15610 format %{ "ShouldNotReachHere" %}
15611
15612 ins_encode %{
15613 if (is_reachable()) {
15614 const char* str = __ code_string(_halt_reason);
15615 __ stop(str);
15616 }
15617 %}
15618
15619 ins_pipe(pipe_class_default);
15620 %}
15621
15622 // ============================================================================
15623 // Partial Subtype Check
15624 //
15625 // superklass array for an instance of the superklass. Set a hidden
15626 // internal cache on a hit (cache is checked with exposed code in
15627 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15628 // encoding ALSO sets flags.
15629
15630 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15631 %{
15632 match(Set result (PartialSubtypeCheck sub super));
15633 predicate(!UseSecondarySupersTable);
15634 effect(KILL cr, KILL temp);
15635
15636 ins_cost(20 * INSN_COST); // slightly larger than the next version
15637 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15638
15639 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15640
15641 opcode(0x1); // Force zero of result reg on hit
15642
15643 ins_pipe(pipe_class_memory);
15644 %}
15645
15646 // Two versions of partialSubtypeCheck, both used when we need to
15647 // search for a super class in the secondary supers array. The first
15648 // is used when we don't know _a priori_ the class being searched
15649 // for. The second, far more common, is used when we do know: this is
15650 // used for instanceof, checkcast, and any case where C2 can determine
15651 // it by constant propagation.
15652
15653 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15654 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15655 rFlagsReg cr)
15656 %{
15657 match(Set result (PartialSubtypeCheck sub super));
15658 predicate(UseSecondarySupersTable);
15659 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15660
15661 ins_cost(10 * INSN_COST); // slightly larger than the next version
15662 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15663
15664 ins_encode %{
15665 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15666 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15667 $vtemp$$FloatRegister,
15668 $result$$Register, /*L_success*/nullptr);
15669 %}
15670
15671 ins_pipe(pipe_class_memory);
15672 %}
15673
15674 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15675 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15676 rFlagsReg cr)
15677 %{
15678 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15679 predicate(UseSecondarySupersTable);
15680 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15681
15682 ins_cost(5 * INSN_COST); // smaller than the next version
15683 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15684
15685 ins_encode %{
15686 bool success = false;
15687 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15688 if (InlineSecondarySupersTest) {
15689 success =
15690 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15691 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15692 $vtemp$$FloatRegister,
15693 $result$$Register,
15694 super_klass_slot);
15695 } else {
15696 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15697 success = (call != nullptr);
15698 }
15699 if (!success) {
15700 ciEnv::current()->record_failure("CodeCache is full");
15701 return;
15702 }
15703 %}
15704
15705 ins_pipe(pipe_class_memory);
15706 %}
15707
15708 // Intrisics for String.compareTo()
15709
15710 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15711 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15712 %{
15713 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15714 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15715 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15716
15717 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15718 ins_encode %{
15719 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15720 __ string_compare($str1$$Register, $str2$$Register,
15721 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15722 $tmp1$$Register, $tmp2$$Register,
15723 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15724 %}
15725 ins_pipe(pipe_class_memory);
15726 %}
15727
15728 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15729 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15730 %{
15731 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15732 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15733 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15734
15735 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15736 ins_encode %{
15737 __ string_compare($str1$$Register, $str2$$Register,
15738 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15739 $tmp1$$Register, $tmp2$$Register,
15740 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15741 %}
15742 ins_pipe(pipe_class_memory);
15743 %}
15744
15745 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15746 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15747 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15748 %{
15749 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15750 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15751 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15752 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15753
15754 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15755 ins_encode %{
15756 __ string_compare($str1$$Register, $str2$$Register,
15757 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15758 $tmp1$$Register, $tmp2$$Register,
15759 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15760 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15761 %}
15762 ins_pipe(pipe_class_memory);
15763 %}
15764
15765 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15766 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15767 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15768 %{
15769 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15770 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15771 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15772 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15773
15774 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15775 ins_encode %{
15776 __ string_compare($str1$$Register, $str2$$Register,
15777 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15778 $tmp1$$Register, $tmp2$$Register,
15779 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15780 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15781 %}
15782 ins_pipe(pipe_class_memory);
15783 %}
15784
15785 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15786 // these string_compare variants as NEON register type for convenience so that the prototype of
15787 // string_compare can be shared with all variants.
15788
15789 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15790 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15791 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15792 pRegGov_P1 pgtmp2, rFlagsReg cr)
15793 %{
15794 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15795 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15796 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15797 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15798
15799 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15800 ins_encode %{
15801 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15802 __ string_compare($str1$$Register, $str2$$Register,
15803 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15804 $tmp1$$Register, $tmp2$$Register,
15805 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15806 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15807 StrIntrinsicNode::LL);
15808 %}
15809 ins_pipe(pipe_class_memory);
15810 %}
15811
15812 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15813 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15814 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15815 pRegGov_P1 pgtmp2, rFlagsReg cr)
15816 %{
15817 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15818 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15819 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15820 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15821
15822 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15823 ins_encode %{
15824 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15825 __ string_compare($str1$$Register, $str2$$Register,
15826 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15827 $tmp1$$Register, $tmp2$$Register,
15828 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15829 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15830 StrIntrinsicNode::LU);
15831 %}
15832 ins_pipe(pipe_class_memory);
15833 %}
15834
15835 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15836 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15837 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15838 pRegGov_P1 pgtmp2, rFlagsReg cr)
15839 %{
15840 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15841 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15842 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15843 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15844
15845 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15846 ins_encode %{
15847 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15848 __ string_compare($str1$$Register, $str2$$Register,
15849 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15850 $tmp1$$Register, $tmp2$$Register,
15851 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15852 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15853 StrIntrinsicNode::UL);
15854 %}
15855 ins_pipe(pipe_class_memory);
15856 %}
15857
15858 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15859 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15860 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15861 pRegGov_P1 pgtmp2, rFlagsReg cr)
15862 %{
15863 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15864 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15865 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15866 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15867
15868 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15869 ins_encode %{
15870 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15871 __ string_compare($str1$$Register, $str2$$Register,
15872 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15873 $tmp1$$Register, $tmp2$$Register,
15874 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15875 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15876 StrIntrinsicNode::UU);
15877 %}
15878 ins_pipe(pipe_class_memory);
15879 %}
15880
15881 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15882 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15883 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15884 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15885 %{
15886 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15887 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15888 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15889 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15890 TEMP vtmp0, TEMP vtmp1, KILL cr);
15891 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15892 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15893
15894 ins_encode %{
15895 __ string_indexof($str1$$Register, $str2$$Register,
15896 $cnt1$$Register, $cnt2$$Register,
15897 $tmp1$$Register, $tmp2$$Register,
15898 $tmp3$$Register, $tmp4$$Register,
15899 $tmp5$$Register, $tmp6$$Register,
15900 -1, $result$$Register, StrIntrinsicNode::UU);
15901 %}
15902 ins_pipe(pipe_class_memory);
15903 %}
15904
15905 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15906 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15907 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15908 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15909 %{
15910 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15911 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15912 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15913 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15914 TEMP vtmp0, TEMP vtmp1, KILL cr);
15915 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15916 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15917
15918 ins_encode %{
15919 __ string_indexof($str1$$Register, $str2$$Register,
15920 $cnt1$$Register, $cnt2$$Register,
15921 $tmp1$$Register, $tmp2$$Register,
15922 $tmp3$$Register, $tmp4$$Register,
15923 $tmp5$$Register, $tmp6$$Register,
15924 -1, $result$$Register, StrIntrinsicNode::LL);
15925 %}
15926 ins_pipe(pipe_class_memory);
15927 %}
15928
15929 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15930 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15931 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15932 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15933 %{
15934 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15935 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15936 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15937 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15938 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15939 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15940 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15941
15942 ins_encode %{
15943 __ string_indexof($str1$$Register, $str2$$Register,
15944 $cnt1$$Register, $cnt2$$Register,
15945 $tmp1$$Register, $tmp2$$Register,
15946 $tmp3$$Register, $tmp4$$Register,
15947 $tmp5$$Register, $tmp6$$Register,
15948 -1, $result$$Register, StrIntrinsicNode::UL);
15949 %}
15950 ins_pipe(pipe_class_memory);
15951 %}
15952
15953 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15954 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15955 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15956 %{
15957 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15958 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15959 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15960 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15961 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15962 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15963
15964 ins_encode %{
15965 int icnt2 = (int)$int_cnt2$$constant;
15966 __ string_indexof($str1$$Register, $str2$$Register,
15967 $cnt1$$Register, zr,
15968 $tmp1$$Register, $tmp2$$Register,
15969 $tmp3$$Register, $tmp4$$Register, zr, zr,
15970 icnt2, $result$$Register, StrIntrinsicNode::UU);
15971 %}
15972 ins_pipe(pipe_class_memory);
15973 %}
15974
15975 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15976 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15977 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15978 %{
15979 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15980 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15981 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15982 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15983 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15984 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15985
15986 ins_encode %{
15987 int icnt2 = (int)$int_cnt2$$constant;
15988 __ string_indexof($str1$$Register, $str2$$Register,
15989 $cnt1$$Register, zr,
15990 $tmp1$$Register, $tmp2$$Register,
15991 $tmp3$$Register, $tmp4$$Register, zr, zr,
15992 icnt2, $result$$Register, StrIntrinsicNode::LL);
15993 %}
15994 ins_pipe(pipe_class_memory);
15995 %}
15996
15997 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15998 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15999 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16000 %{
16001 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16002 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16003 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16004 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16005 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16006 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16007
16008 ins_encode %{
16009 int icnt2 = (int)$int_cnt2$$constant;
16010 __ string_indexof($str1$$Register, $str2$$Register,
16011 $cnt1$$Register, zr,
16012 $tmp1$$Register, $tmp2$$Register,
16013 $tmp3$$Register, $tmp4$$Register, zr, zr,
16014 icnt2, $result$$Register, StrIntrinsicNode::UL);
16015 %}
16016 ins_pipe(pipe_class_memory);
16017 %}
16018
16019 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16020 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16021 iRegINoSp tmp3, rFlagsReg cr)
16022 %{
16023 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16024 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16025 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16026 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16027
16028 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16029
16030 ins_encode %{
16031 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16032 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16033 $tmp3$$Register);
16034 %}
16035 ins_pipe(pipe_class_memory);
16036 %}
16037
16038 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16039 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16040 iRegINoSp tmp3, rFlagsReg cr)
16041 %{
16042 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16043 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16044 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16045 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16046
16047 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16048
16049 ins_encode %{
16050 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16051 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16052 $tmp3$$Register);
16053 %}
16054 ins_pipe(pipe_class_memory);
16055 %}
16056
16057 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16058 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16059 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16060 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16061 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16062 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16063 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16064 ins_encode %{
16065 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16066 $result$$Register, $ztmp1$$FloatRegister,
16067 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16068 $ptmp$$PRegister, true /* isL */);
16069 %}
16070 ins_pipe(pipe_class_memory);
16071 %}
16072
16073 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16074 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16075 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16076 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16077 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16078 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16079 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16080 ins_encode %{
16081 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16082 $result$$Register, $ztmp1$$FloatRegister,
16083 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16084 $ptmp$$PRegister, false /* isL */);
16085 %}
16086 ins_pipe(pipe_class_memory);
16087 %}
16088
16089 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16090 iRegI_R0 result, rFlagsReg cr)
16091 %{
16092 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16093 match(Set result (StrEquals (Binary str1 str2) cnt));
16094 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16095
16096 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16097 ins_encode %{
16098 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16099 __ string_equals($str1$$Register, $str2$$Register,
16100 $result$$Register, $cnt$$Register);
16101 %}
16102 ins_pipe(pipe_class_memory);
16103 %}
16104
16105 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16106 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16107 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16108 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16109 iRegP_R10 tmp, rFlagsReg cr)
16110 %{
16111 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16112 match(Set result (AryEq ary1 ary2));
16113 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16114 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16115 TEMP vtmp6, TEMP vtmp7, KILL cr);
16116
16117 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16118 ins_encode %{
16119 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16120 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16121 $result$$Register, $tmp$$Register, 1);
16122 if (tpc == nullptr) {
16123 ciEnv::current()->record_failure("CodeCache is full");
16124 return;
16125 }
16126 %}
16127 ins_pipe(pipe_class_memory);
16128 %}
16129
16130 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16131 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16132 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16133 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16134 iRegP_R10 tmp, rFlagsReg cr)
16135 %{
16136 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16137 match(Set result (AryEq ary1 ary2));
16138 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16139 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16140 TEMP vtmp6, TEMP vtmp7, KILL cr);
16141
16142 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16143 ins_encode %{
16144 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16145 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16146 $result$$Register, $tmp$$Register, 2);
16147 if (tpc == nullptr) {
16148 ciEnv::current()->record_failure("CodeCache is full");
16149 return;
16150 }
16151 %}
16152 ins_pipe(pipe_class_memory);
16153 %}
16154
16155 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16156 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16157 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16158 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16159 %{
16160 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16161 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16162 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16163
16164 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16165 ins_encode %{
16166 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16167 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16168 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16169 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16170 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16171 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16172 (BasicType)$basic_type$$constant);
16173 if (tpc == nullptr) {
16174 ciEnv::current()->record_failure("CodeCache is full");
16175 return;
16176 }
16177 %}
16178 ins_pipe(pipe_class_memory);
16179 %}
16180
16181 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16182 %{
16183 match(Set result (CountPositives ary1 len));
16184 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16185 format %{ "count positives byte[] $ary1,$len -> $result" %}
16186 ins_encode %{
16187 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16188 if (tpc == nullptr) {
16189 ciEnv::current()->record_failure("CodeCache is full");
16190 return;
16191 }
16192 %}
16193 ins_pipe( pipe_slow );
16194 %}
16195
16196 // fast char[] to byte[] compression
16197 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16198 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16199 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16200 iRegI_R0 result, rFlagsReg cr)
16201 %{
16202 match(Set result (StrCompressedCopy src (Binary dst len)));
16203 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16204 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16205
16206 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16207 ins_encode %{
16208 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16209 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16210 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16211 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16212 %}
16213 ins_pipe(pipe_slow);
16214 %}
16215
16216 // fast byte[] to char[] inflation
16217 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16218 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16219 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16220 %{
16221 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16222 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16223 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16224 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16225
16226 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16227 ins_encode %{
16228 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16229 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16230 $vtmp2$$FloatRegister, $tmp$$Register);
16231 if (tpc == nullptr) {
16232 ciEnv::current()->record_failure("CodeCache is full");
16233 return;
16234 }
16235 %}
16236 ins_pipe(pipe_class_memory);
16237 %}
16238
16239 // encode char[] to byte[] in ISO_8859_1
16240 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16241 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16242 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16243 iRegI_R0 result, rFlagsReg cr)
16244 %{
16245 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16246 match(Set result (EncodeISOArray src (Binary dst len)));
16247 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16248 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16249
16250 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16251 ins_encode %{
16252 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16253 $result$$Register, false,
16254 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16255 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16256 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16257 %}
16258 ins_pipe(pipe_class_memory);
16259 %}
16260
16261 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16262 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16263 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16264 iRegI_R0 result, rFlagsReg cr)
16265 %{
16266 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16267 match(Set result (EncodeISOArray src (Binary dst len)));
16268 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16269 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16270
16271 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16272 ins_encode %{
16273 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16274 $result$$Register, true,
16275 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16276 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16277 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16278 %}
16279 ins_pipe(pipe_class_memory);
16280 %}
16281
16282 //----------------------------- CompressBits/ExpandBits ------------------------
16283
16284 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16285 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16286 match(Set dst (CompressBits src mask));
16287 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16288 format %{ "mov $tsrc, $src\n\t"
16289 "mov $tmask, $mask\n\t"
16290 "bext $tdst, $tsrc, $tmask\n\t"
16291 "mov $dst, $tdst"
16292 %}
16293 ins_encode %{
16294 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16295 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16296 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16297 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16298 %}
16299 ins_pipe(pipe_slow);
16300 %}
16301
16302 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16303 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16304 match(Set dst (CompressBits (LoadI mem) mask));
16305 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16306 format %{ "ldrs $tsrc, $mem\n\t"
16307 "ldrs $tmask, $mask\n\t"
16308 "bext $tdst, $tsrc, $tmask\n\t"
16309 "mov $dst, $tdst"
16310 %}
16311 ins_encode %{
16312 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16313 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16314 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16315 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16316 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16317 %}
16318 ins_pipe(pipe_slow);
16319 %}
16320
16321 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16322 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16323 match(Set dst (CompressBits src mask));
16324 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16325 format %{ "mov $tsrc, $src\n\t"
16326 "mov $tmask, $mask\n\t"
16327 "bext $tdst, $tsrc, $tmask\n\t"
16328 "mov $dst, $tdst"
16329 %}
16330 ins_encode %{
16331 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16332 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16333 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16334 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16335 %}
16336 ins_pipe(pipe_slow);
16337 %}
16338
16339 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16340 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16341 match(Set dst (CompressBits (LoadL mem) mask));
16342 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16343 format %{ "ldrd $tsrc, $mem\n\t"
16344 "ldrd $tmask, $mask\n\t"
16345 "bext $tdst, $tsrc, $tmask\n\t"
16346 "mov $dst, $tdst"
16347 %}
16348 ins_encode %{
16349 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16350 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16351 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16352 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16353 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16354 %}
16355 ins_pipe(pipe_slow);
16356 %}
16357
16358 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16359 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16360 match(Set dst (ExpandBits src mask));
16361 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16362 format %{ "mov $tsrc, $src\n\t"
16363 "mov $tmask, $mask\n\t"
16364 "bdep $tdst, $tsrc, $tmask\n\t"
16365 "mov $dst, $tdst"
16366 %}
16367 ins_encode %{
16368 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16369 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16370 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16371 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16372 %}
16373 ins_pipe(pipe_slow);
16374 %}
16375
16376 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16377 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16378 match(Set dst (ExpandBits (LoadI mem) mask));
16379 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16380 format %{ "ldrs $tsrc, $mem\n\t"
16381 "ldrs $tmask, $mask\n\t"
16382 "bdep $tdst, $tsrc, $tmask\n\t"
16383 "mov $dst, $tdst"
16384 %}
16385 ins_encode %{
16386 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16387 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16388 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16389 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16390 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16391 %}
16392 ins_pipe(pipe_slow);
16393 %}
16394
16395 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16396 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16397 match(Set dst (ExpandBits src mask));
16398 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16399 format %{ "mov $tsrc, $src\n\t"
16400 "mov $tmask, $mask\n\t"
16401 "bdep $tdst, $tsrc, $tmask\n\t"
16402 "mov $dst, $tdst"
16403 %}
16404 ins_encode %{
16405 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16406 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16407 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16408 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16409 %}
16410 ins_pipe(pipe_slow);
16411 %}
16412
16413
16414 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16415 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16416 match(Set dst (ExpandBits (LoadL mem) mask));
16417 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16418 format %{ "ldrd $tsrc, $mem\n\t"
16419 "ldrd $tmask, $mask\n\t"
16420 "bdep $tdst, $tsrc, $tmask\n\t"
16421 "mov $dst, $tdst"
16422 %}
16423 ins_encode %{
16424 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16425 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16426 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16427 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16428 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16429 %}
16430 ins_pipe(pipe_slow);
16431 %}
16432
16433 //----------------------------- Reinterpret ----------------------------------
16434 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16435 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16436 match(Set dst (ReinterpretHF2S src));
16437 format %{ "reinterpretHF2S $dst, $src" %}
16438 ins_encode %{
16439 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16440 %}
16441 ins_pipe(pipe_slow);
16442 %}
16443
16444 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16445 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16446 match(Set dst (ReinterpretS2HF src));
16447 format %{ "reinterpretS2HF $dst, $src" %}
16448 ins_encode %{
16449 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16450 %}
16451 ins_pipe(pipe_slow);
16452 %}
16453
16454 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16455 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16456 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16457 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16458 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16459 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16460 // can be omitted in this pattern, resulting in -
16461 // fcvt $dst, $src // Convert float to half-precision float
16462 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16463 %{
16464 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16465 format %{ "convF2HFAndS2HF $dst, $src" %}
16466 ins_encode %{
16467 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16468 %}
16469 ins_pipe(pipe_slow);
16470 %}
16471
16472 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16473 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16474 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16475 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16476 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16477 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16478 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16479 // resulting in -
16480 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16481 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16482 %{
16483 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16484 format %{ "convHF2SAndHF2F $dst, $src" %}
16485 ins_encode %{
16486 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16487 %}
16488 ins_pipe(pipe_slow);
16489 %}
16490
16491 // ============================================================================
16492 // This name is KNOWN by the ADLC and cannot be changed.
16493 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16494 // for this guy.
16495 instruct tlsLoadP(thread_RegP dst)
16496 %{
16497 match(Set dst (ThreadLocal));
16498
16499 ins_cost(0);
16500
16501 format %{ " -- \t// $dst=Thread::current(), empty" %}
16502
16503 size(0);
16504
16505 ins_encode( /*empty*/ );
16506
16507 ins_pipe(pipe_class_empty);
16508 %}
16509
16510 //----------PEEPHOLE RULES-----------------------------------------------------
16511 // These must follow all instruction definitions as they use the names
16512 // defined in the instructions definitions.
16513 //
16514 // peepmatch ( root_instr_name [preceding_instruction]* );
16515 //
16516 // peepconstraint %{
16517 // (instruction_number.operand_name relational_op instruction_number.operand_name
16518 // [, ...] );
16519 // // instruction numbers are zero-based using left to right order in peepmatch
16520 //
16521 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16522 // // provide an instruction_number.operand_name for each operand that appears
16523 // // in the replacement instruction's match rule
16524 //
16525 // ---------VM FLAGS---------------------------------------------------------
16526 //
16527 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16528 //
16529 // Each peephole rule is given an identifying number starting with zero and
16530 // increasing by one in the order seen by the parser. An individual peephole
16531 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16532 // on the command-line.
16533 //
16534 // ---------CURRENT LIMITATIONS----------------------------------------------
16535 //
16536 // Only match adjacent instructions in same basic block
16537 // Only equality constraints
16538 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16539 // Only one replacement instruction
16540 //
16541 // ---------EXAMPLE----------------------------------------------------------
16542 //
16543 // // pertinent parts of existing instructions in architecture description
16544 // instruct movI(iRegINoSp dst, iRegI src)
16545 // %{
16546 // match(Set dst (CopyI src));
16547 // %}
16548 //
16549 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16550 // %{
16551 // match(Set dst (AddI dst src));
16552 // effect(KILL cr);
16553 // %}
16554 //
16555 // // Change (inc mov) to lea
16556 // peephole %{
16557 // // increment preceded by register-register move
16558 // peepmatch ( incI_iReg movI );
16559 // // require that the destination register of the increment
16560 // // match the destination register of the move
16561 // peepconstraint ( 0.dst == 1.dst );
16562 // // construct a replacement instruction that sets
16563 // // the destination to ( move's source register + one )
16564 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16565 // %}
16566 //
16567
16568 // Implementation no longer uses movX instructions since
16569 // machine-independent system no longer uses CopyX nodes.
16570 //
16571 // peephole
16572 // %{
16573 // peepmatch (incI_iReg movI);
16574 // peepconstraint (0.dst == 1.dst);
16575 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16576 // %}
16577
16578 // peephole
16579 // %{
16580 // peepmatch (decI_iReg movI);
16581 // peepconstraint (0.dst == 1.dst);
16582 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16583 // %}
16584
16585 // peephole
16586 // %{
16587 // peepmatch (addI_iReg_imm movI);
16588 // peepconstraint (0.dst == 1.dst);
16589 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16590 // %}
16591
16592 // peephole
16593 // %{
16594 // peepmatch (incL_iReg movL);
16595 // peepconstraint (0.dst == 1.dst);
16596 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16597 // %}
16598
16599 // peephole
16600 // %{
16601 // peepmatch (decL_iReg movL);
16602 // peepconstraint (0.dst == 1.dst);
16603 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16604 // %}
16605
16606 // peephole
16607 // %{
16608 // peepmatch (addL_iReg_imm movL);
16609 // peepconstraint (0.dst == 1.dst);
16610 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16611 // %}
16612
16613 // peephole
16614 // %{
16615 // peepmatch (addP_iReg_imm movP);
16616 // peepconstraint (0.dst == 1.dst);
16617 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16618 // %}
16619
16620 // // Change load of spilled value to only a spill
16621 // instruct storeI(memory mem, iRegI src)
16622 // %{
16623 // match(Set mem (StoreI mem src));
16624 // %}
16625 //
16626 // instruct loadI(iRegINoSp dst, memory mem)
16627 // %{
16628 // match(Set dst (LoadI mem));
16629 // %}
16630 //
16631
16632 //----------SMARTSPILL RULES---------------------------------------------------
16633 // These must follow all instruction definitions as they use the names
16634 // defined in the instructions definitions.
16635
16636 // Local Variables:
16637 // mode: c++
16638 // End: